From 12e2b7538a1bf9f4b953444f10132c401b4a6e60 Mon Sep 17 00:00:00 2001 From: yxia216 Date: Thu, 8 Oct 2026 09:43:16 -0400 Subject: [PATCH 1/2] init for decision api Signed-off-by: yxia216 --- cmd/extproc/mainlib/main.go | 3 + internal/apischema/openai/decisions.go | 67 +++++++++ internal/endpointspec/endpointspec.go | 52 +++++++ internal/endpointspec/endpointspec_test.go | 67 +++++++++ internal/metrics/genai.go | 1 + .../tracing/openinference/openai/decisions.go | 97 +++++++++++++ .../openinference/openai/decisions_test.go | 130 ++++++++++++++++++ internal/tracing/otelgenai/backend_test.go | 1 + internal/tracing/otelgenai/coverage_test.go | 7 + internal/tracing/otelgenai/endpoints.go | 24 ++++ internal/tracing/otelgenai/endpoints_test.go | 46 +++++++ internal/tracing/otelgenai/otelgenai.go | 1 + internal/tracing/semconv.go | 3 + internal/tracing/semconv_test.go | 1 + internal/tracing/span.go | 1 + internal/tracing/tracer.go | 14 ++ internal/tracing/tracing.go | 12 ++ internal/tracing/tracing_test.go | 1 + internal/tracing/tracingapi/api.go | 15 ++ internal/translator/openai_decisions.go | 93 +++++++++++++ internal/translator/openai_decisions_test.go | 125 +++++++++++++++++ internal/translator/translator.go | 2 + .../llm-integrations/supported-endpoints.md | 44 ++++++ tests/data-plane/testupstream_test.go | 11 ++ 24 files changed, 818 insertions(+) create mode 100644 internal/apischema/openai/decisions.go create mode 100644 internal/tracing/openinference/openai/decisions.go create mode 100644 internal/tracing/openinference/openai/decisions_test.go create mode 100644 internal/translator/openai_decisions.go create mode 100644 internal/translator/openai_decisions_test.go diff --git a/cmd/extproc/mainlib/main.go b/cmd/extproc/mainlib/main.go index ceed670691..9830c99b50 100644 --- a/cmd/extproc/mainlib/main.go +++ b/cmd/extproc/mainlib/main.go @@ -309,6 +309,7 @@ func Main(ctx context.Context, args []string, stderr io.Writer) (err error) { embeddingsMetricsFactory := metrics.NewMetricsFactory(meter, metricsRequestHeaderAttributes, metrics.GenAIOperationEmbedding) imageGenerationMetricsFactory := metrics.NewMetricsFactory(meter, metricsRequestHeaderAttributes, metrics.GenAIOperationImageGeneration) responsesMetricsFactory := metrics.NewMetricsFactory(meter, metricsRequestHeaderAttributes, metrics.GenAIOperationResponses) + decisionsMetricsFactory := metrics.NewMetricsFactory(meter, metricsRequestHeaderAttributes, metrics.GenAIOperationDecisions) speechMetricsFactory := metrics.NewMetricsFactory(meter, metricsRequestHeaderAttributes, metrics.GenAIOperationSpeech) transcriptionMetricsFactory := metrics.NewMetricsFactory(meter, metricsRequestHeaderAttributes, metrics.GenAIOperationTranscription) translationMetricsFactory := metrics.NewMetricsFactory(meter, metricsRequestHeaderAttributes, metrics.GenAIOperationTranslation) @@ -333,6 +334,8 @@ func Main(ctx context.Context, args []string, stderr io.Writer) (err error) { embeddingsMetricsFactory, tracing.EmbeddingsTracer(), endpointspec.EmbeddingsEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/responses"), extproc.NewFactory( responsesMetricsFactory, tracing.ResponsesTracer(), endpointspec.ResponsesEndpointSpec{})) + server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/decisions"), extproc.NewFactory( + decisionsMetricsFactory, tracing.DecisionsTracer(), endpointspec.DecisionsEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/responses/input_tokens"), extproc.NewFactory( responsesInputTokensMetricsFactory, tracing.ResponsesInputTokensTracer(), endpointspec.ResponsesInputTokensEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/audio/speech"), extproc.NewFactory( diff --git a/internal/apischema/openai/decisions.go b/internal/apischema/openai/decisions.go new file mode 100644 index 0000000000..9ef997e198 --- /dev/null +++ b/internal/apischema/openai/decisions.go @@ -0,0 +1,67 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package openai + +import "github.com/envoyproxy/ai-gateway/internal/json" + +// DecisionRequest represents the request body for POST /v1/decisions. +// Input is kept as raw JSON because the API accepts either a string or an array +// of user messages containing text and inline base64 images. +type DecisionRequest struct { + Model string `json:"model"` + Input json.RawMessage `json:"input"` + Questions []DecisionQuestion `json:"questions"` +} + +// DecisionQuestion is a predicate, choice, or score question. +type DecisionQuestion struct { + Type string `json:"type"` + Name string `json:"name"` + Instructions string `json:"instructions"` + Choices []DecisionChoiceOption `json:"choices,omitempty"` + Levels []DecisionScoreLevel `json:"levels,omitempty"` +} + +// DecisionChoiceOption is one allowed result for a choice question. +type DecisionChoiceOption struct { + Value json.RawMessage `json:"value"` + Description string `json:"description"` +} + +// DecisionScoreLevel is one ordered level for a score question. +type DecisionScoreLevel struct { + Label string `json:"label"` + Description string `json:"description"` +} + +// DecisionResponse represents a response from POST /v1/decisions. +type DecisionResponse struct { + Answers []DecisionAnswer `json:"answers"` + Model string `json:"model,omitempty"` + Usage *ResponseUsage `json:"usage,omitempty"` +} + +// DecisionAnswer is a predicate, choice, score, or refusal answer. Pointer +// numeric fields preserve valid zero values while omitting fields belonging to +// other variants. +type DecisionAnswer struct { + Type string `json:"type"` + Name string `json:"name"` + Probability *float64 `json:"probability,omitempty"` + Choice json.RawMessage `json:"choice,omitempty"` + Probabilities []DecisionProbability `json:"probabilities,omitempty"` + Confidence *float64 `json:"confidence,omitempty"` + Score *float64 `json:"score,omitempty"` +} + +// DecisionProbability is one entry in a choice or score probability +// distribution. Value is raw because choices use strings or booleans, while +// score levels use numeric indices. +type DecisionProbability struct { + Value json.RawMessage `json:"value"` + Label string `json:"label,omitempty"` + Probability float64 `json:"probability"` +} diff --git a/internal/endpointspec/endpointspec.go b/internal/endpointspec/endpointspec.go index 37e8cbe868..790cd2599b 100644 --- a/internal/endpointspec/endpointspec.go +++ b/internal/endpointspec/endpointspec.go @@ -107,6 +107,8 @@ type ( ImageGenerationEndpointSpec struct{} // ResponsesEndpointSpec implements EndpointSpec for /v1/responses. ResponsesEndpointSpec struct{} + // DecisionsEndpointSpec implements EndpointSpec for /v1/decisions. + DecisionsEndpointSpec struct{} // MessagesEndpointSpec implements EndpointSpec for /v1/messages. MessagesEndpointSpec struct{} // RerankEndpointSpec implements EndpointSpec for /v2/rerank. @@ -416,6 +418,56 @@ func (ResponsesEndpointSpec) RedactSensitiveInfoFromRequest(req *openai.Response return &redacted, nil } +// ParseBody implements [EndpointSpec.ParseBody]. Decisions is a non-streaming endpoint. +func (DecisionsEndpointSpec) ParseBody(body []byte, _ bool) (internalapi.OriginalModel, *openai.DecisionRequest, bool, []byte, error) { + var req openai.DecisionRequest + if err := json.Unmarshal(body, &req); err != nil { + return "", nil, false, nil, fmt.Errorf("%w: failed to parse JSON for /v1/decisions: %w", internalapi.ErrMalformedRequest, err) + } + return req.Model, &req, false, nil, nil +} + +func (DecisionsEndpointSpec) ParseMultipartBody([]byte, string, bool) (internalapi.OriginalModel, *openai.DecisionRequest, bool, []byte, error) { + return "", nil, false, nil, errMultipartNotSupported +} + +func (DecisionsEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.OpenAIDecisionsTranslator, error) { + switch schema.Name { + case filterapi.APISchemaOpenAI: + return translator.NewDecisionsOpenAIToOpenAITranslator(schema.OpenAIPrefix(), modelNameOverride), nil + default: + return nil, fmt.Errorf("%w: unsupported API schema: backend=%s", internalapi.ErrInvalidRequestBody, schema) + } +} + +// RedactSensitiveInfoFromRequest removes evidence and rubric content while +// retaining the model, question names, and discriminators needed for debugging. +func (DecisionsEndpointSpec) RedactSensitiveInfoFromRequest(req *openai.DecisionRequest) (*openai.DecisionRequest, error) { + redacted := *req + redacted.Input = redactRawJSON(req.Input) + redacted.Questions = make([]openai.DecisionQuestion, len(req.Questions)) + for i := range req.Questions { + q := req.Questions[i] + q.Instructions = redaction.RedactString(q.Instructions) + q.Choices = make([]openai.DecisionChoiceOption, len(req.Questions[i].Choices)) + for j := range req.Questions[i].Choices { + q.Choices[j] = openai.DecisionChoiceOption{ + Value: redactRawJSON(req.Questions[i].Choices[j].Value), + Description: redaction.RedactString(req.Questions[i].Choices[j].Description), + } + } + q.Levels = make([]openai.DecisionScoreLevel, len(req.Questions[i].Levels)) + for j := range req.Questions[i].Levels { + q.Levels[j] = openai.DecisionScoreLevel{ + Label: redaction.RedactString(req.Questions[i].Levels[j].Label), + Description: redaction.RedactString(req.Questions[i].Levels[j].Description), + } + } + redacted.Questions[i] = q + } + return &redacted, nil +} + // ParseBody implements [EndpointSpec.ParseBody]. func (MessagesEndpointSpec) ParseBody( body []byte, diff --git a/internal/endpointspec/endpointspec_test.go b/internal/endpointspec/endpointspec_test.go index b5644c1a1d..76b9368bdd 100644 --- a/internal/endpointspec/endpointspec_test.go +++ b/internal/endpointspec/endpointspec_test.go @@ -506,6 +506,73 @@ func TestResponsesEndpointSpec_GetTranslator(t *testing.T) { require.ErrorContains(t, err, "unsupported API schema") } +func TestDecisionsEndpointSpec(t *testing.T) { + spec := DecisionsEndpointSpec{} + + t.Run("parse string input", func(t *testing.T) { + body := []byte(`{"model":"gpt-6-luna","input":"charged twice","questions":[{"type":"choice","name":"department","instructions":"route it","choices":[{"value":"billing","description":"payment issues"}]}]}`) + model, req, stream, mutated, err := spec.ParseBody(body, false) + require.NoError(t, err) + require.Equal(t, "gpt-6-luna", model) + require.False(t, stream) + require.Nil(t, mutated) + require.JSONEq(t, `"charged twice"`, string(req.Input)) + require.JSONEq(t, `"billing"`, string(req.Questions[0].Choices[0].Value)) + }) + + t.Run("parse boolean choice values", func(t *testing.T) { + body := []byte(`{"model":"gpt-6-luna","input":"confirm","questions":[{"type":"choice","instructions":"answer it","choices":[{"value":true},{"value":false}]}]}`) + _, req, _, _, err := spec.ParseBody(body, false) + require.NoError(t, err) + require.JSONEq(t, `true`, string(req.Questions[0].Choices[0].Value)) + require.JSONEq(t, `false`, string(req.Questions[0].Choices[1].Value)) + }) + + t.Run("parse image input", func(t *testing.T) { + body := []byte(`{"model":"gpt-6-luna","input":[{"role":"user","content":[{"type":"input_text","text":"inspect"},{"type":"input_image","image_url":"data:image/png;base64,AAAA"}]}],"questions":[{"type":"predicate","name":"damage","instructions":"visible damage?"}]}`) + _, req, _, _, err := spec.ParseBody(body, false) + require.NoError(t, err) + require.JSONEq(t, `[{"role":"user","content":[{"type":"input_text","text":"inspect"},{"type":"input_image","image_url":"data:image/png;base64,AAAA"}]}]`, string(req.Input)) + }) + + t.Run("invalid json", func(t *testing.T) { + _, _, _, _, err := spec.ParseBody([]byte("{"), false) + require.ErrorIs(t, err, internalapi.ErrMalformedRequest) + require.ErrorContains(t, err, "/v1/decisions") + }) + + t.Run("multipart unsupported", func(t *testing.T) { + _, _, _, _, err := spec.ParseMultipartBody(nil, "multipart/form-data", false) + require.ErrorIs(t, err, errMultipartNotSupported) + }) + + t.Run("translator support", func(t *testing.T) { + _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + require.NoError(t, err) + + _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaTypeSafe}, "override") + require.ErrorIs(t, err, internalapi.ErrInvalidRequestBody) + }) +} + +func TestDecisionsEndpointSpec_RedactSensitiveInfoFromRequest(t *testing.T) { + body := []byte(`{"model":"gpt-6-luna","input":"customer complaint","questions":[{"type":"choice","name":"department","instructions":"route this complaint","choices":[{"value":"billing","description":"payment issues"}]},{"type":"score","name":"severity","instructions":"score it","levels":[{"label":"critical","description":"no workaround"}]}]}`) + _, req, _, _, err := DecisionsEndpointSpec{}.ParseBody(body, false) + require.NoError(t, err) + redacted, err := DecisionsEndpointSpec{}.RedactSensitiveInfoFromRequest(req) + require.NoError(t, err) + + encoded, err := json.Marshal(redacted) + require.NoError(t, err) + text := string(encoded) + require.Contains(t, text, "[REDACTED LENGTH=") + require.NotContains(t, text, "customer complaint") + require.NotContains(t, text, "payment issues") + require.NotContains(t, text, "critical") + require.Contains(t, text, `"model":"gpt-6-luna"`) + require.Contains(t, text, `"name":"department"`) +} + func TestTokenizeEndpointSpec_ParseBody(t *testing.T) { spec := TokenizeEndpointSpec{} diff --git a/internal/metrics/genai.go b/internal/metrics/genai.go index 1fa754b8d6..ad73709f34 100644 --- a/internal/metrics/genai.go +++ b/internal/metrics/genai.go @@ -31,6 +31,7 @@ const ( GenAIOperationMessages GenAIOperation = "messages" GenAIOperationImageGeneration GenAIOperation = "image_generation" GenAIOperationResponses GenAIOperation = "responses" + GenAIOperationDecisions GenAIOperation = "decisions" GenAIOperationSpeech GenAIOperation = "speech" GenAIOperationTranscription GenAIOperation = "transcription" GenAIOperationTranslation GenAIOperation = "translation" diff --git a/internal/tracing/openinference/openai/decisions.go b/internal/tracing/openinference/openai/decisions.go new file mode 100644 index 0000000000..42c8429d0f --- /dev/null +++ b/internal/tracing/openinference/openai/decisions.go @@ -0,0 +1,97 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package openai + +import ( + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + openaischema "github.com/envoyproxy/ai-gateway/internal/apischema/openai" + "github.com/envoyproxy/ai-gateway/internal/json" + "github.com/envoyproxy/ai-gateway/internal/tracing/openinference" + "github.com/envoyproxy/ai-gateway/internal/tracing/tracingapi" +) + +// DecisionsRecorder records OpenInference attributes for Decisions requests. +type DecisionsRecorder struct { + tracingapi.NoopChunkRecorder[struct{}] + traceConfig *openinference.TraceConfig +} + +func NewDecisionsRecorder(config *openinference.TraceConfig) tracingapi.DecisionsRecorder { + if config == nil { + config = openinference.NewTraceConfigFromEnv() + } + return &DecisionsRecorder{traceConfig: config} +} + +func (*DecisionsRecorder) StartParams(*openaischema.DecisionRequest, []byte) (string, []trace.SpanStartOption) { + return "Decisions", []trace.SpanStartOption{trace.WithSpanKind(trace.SpanKindInternal)} +} + +func (r *DecisionsRecorder) RecordRequest(span trace.Span, req *openaischema.DecisionRequest, body []byte) { + attrs := []attribute.KeyValue{ + attribute.String(openinference.LLMSystem, openinference.LLMSystemOpenAI), + attribute.String(openinference.SpanKind, openinference.SpanKindLLM), + } + if req.Model != "" { + attrs = append(attrs, attribute.String(openinference.LLMModelName, req.Model)) + } + if r.traceConfig.HideInputs { + attrs = append(attrs, attribute.String(openinference.InputValue, openinference.RedactedValue)) + } else { + modifiedBody, err := redactImageFromResponseRequestParameters( + body, r.traceConfig.HideInputImages, r.traceConfig.Base64ImageMaxLength, + ) + if err == nil { + attrs = append(attrs, attribute.String(openinference.InputValue, string(modifiedBody))) + } + attrs = append(attrs, attribute.String(openinference.InputMimeType, openinference.MimeTypeJSON)) + } + span.SetAttributes(attrs...) +} + +func (*DecisionsRecorder) RecordResponseOnError(span trace.Span, statusCode int, body []byte) { + openinference.RecordResponseError(span, statusCode, string(body)) +} + +func (r *DecisionsRecorder) RecordResponse(span trace.Span, resp *openaischema.DecisionResponse) { + body := openinference.RedactedValue + attrs := []attribute.KeyValue{} + if resp.Model != "" { + attrs = append(attrs, attribute.String(openinference.LLMModelName, resp.Model)) + } + if u := resp.Usage; u != nil { + if u.InputTokens > 0 { + attrs = append(attrs, attribute.Int(openinference.LLMTokenCountPrompt, int(u.InputTokens))) + } + if u.OutputTokens > 0 { + attrs = append(attrs, attribute.Int(openinference.LLMTokenCountCompletion, int(u.OutputTokens))) + } + if u.TotalTokens > 0 { + attrs = append(attrs, attribute.Int(openinference.LLMTokenCountTotal, int(u.TotalTokens))) + } + if u.InputTokensDetails.CachedTokens > 0 { + attrs = append(attrs, attribute.Int(openinference.LLMTokenCountPromptCacheHit, int(u.InputTokensDetails.CachedTokens))) + } + if cacheWriteTokens := u.InputTokensDetails.CacheWriteTokensValue(); cacheWriteTokens > 0 { + attrs = append(attrs, attribute.Int(openinference.LLMTokenCountPromptCacheWrite, int(cacheWriteTokens))) + } + if u.OutputTokensDetails.ReasoningTokens > 0 { + attrs = append(attrs, attribute.Int(openinference.LLMTokenCountCompletionReasoning, int(u.OutputTokensDetails.ReasoningTokens))) + } + } + if !r.traceConfig.HideOutputs { + if marshaled, err := json.Marshal(resp); err == nil { + body = string(marshaled) + } + attrs = append(attrs, attribute.String(openinference.OutputMimeType, openinference.MimeTypeJSON)) + } + attrs = append(attrs, attribute.String(openinference.OutputValue, body)) + span.SetAttributes(attrs...) + span.SetStatus(codes.Ok, "") +} diff --git a/internal/tracing/openinference/openai/decisions_test.go b/internal/tracing/openinference/openai/decisions_test.go new file mode 100644 index 0000000000..116ee7d284 --- /dev/null +++ b/internal/tracing/openinference/openai/decisions_test.go @@ -0,0 +1,130 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package openai + +import ( + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + oteltrace "go.opentelemetry.io/otel/trace" + + openaischema "github.com/envoyproxy/ai-gateway/internal/apischema/openai" + "github.com/envoyproxy/ai-gateway/internal/json" + "github.com/envoyproxy/ai-gateway/internal/testing/testotel" + "github.com/envoyproxy/ai-gateway/internal/tracing/openinference" +) + +func TestDecisionsRecorder_RecordRequest(t *testing.T) { + req := &openaischema.DecisionRequest{Model: "gpt-6-luna"} + + tests := []struct { + name string + body string + config *openinference.TraceConfig + expected []attribute.KeyValue + }{ + { + name: "records native request", + body: `{"model":"gpt-6-luna","input":"sensitive evidence","questions":[]}`, + config: &openinference.TraceConfig{}, + expected: []attribute.KeyValue{ + attribute.String(openinference.LLMSystem, openinference.LLMSystemOpenAI), + attribute.String(openinference.SpanKind, openinference.SpanKindLLM), + attribute.String(openinference.LLMModelName, "gpt-6-luna"), + attribute.String(openinference.InputValue, `{"model":"gpt-6-luna","input":"sensitive evidence","questions":[]}`), + attribute.String(openinference.InputMimeType, openinference.MimeTypeJSON), + }, + }, + { + name: "redacts input", + body: `{"model":"gpt-6-luna","input":"sensitive evidence","questions":[]}`, + config: &openinference.TraceConfig{HideInputs: true}, + expected: []attribute.KeyValue{ + attribute.String(openinference.LLMSystem, openinference.LLMSystemOpenAI), + attribute.String(openinference.SpanKind, openinference.SpanKindLLM), + attribute.String(openinference.LLMModelName, "gpt-6-luna"), + attribute.String(openinference.InputValue, openinference.RedactedValue), + }, + }, + { + name: "redacts inline input images", + body: `{"model":"gpt-6-luna","input":[{"role":"user","content":[{"type":"input_text","text":"classify this"},{"type":"input_image","image_url":"data:image/png;base64,c2Vuc2l0aXZl"}]}],"questions":[]}`, + config: &openinference.TraceConfig{HideInputImages: true}, + expected: []attribute.KeyValue{ + attribute.String(openinference.LLMSystem, openinference.LLMSystemOpenAI), + attribute.String(openinference.SpanKind, openinference.SpanKindLLM), + attribute.String(openinference.LLMModelName, "gpt-6-luna"), + attribute.String(openinference.InputValue, `{"model":"gpt-6-luna","input":[{"role":"user","content":[{"type":"input_text","text":"classify this"},{"type":"input_image","image_url":"__REDACTED__"}]}],"questions":[]}`), + attribute.String(openinference.InputMimeType, openinference.MimeTypeJSON), + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + r := NewDecisionsRecorder(tc.config) + span := testotel.RecordWithSpan(t, func(span oteltrace.Span) bool { + r.RecordRequest(span, req, []byte(tc.body)) + return false + }) + openinference.RequireAttributesEqual(t, tc.expected, span.Attributes) + }) + } +} + +func TestDecisionsRecorder_RecordResponse(t *testing.T) { + resp := &openaischema.DecisionResponse{ + Answers: []openaischema.DecisionAnswer{{Type: "choice", Name: "route", Choice: json.RawMessage(`"billing"`)}}, + Model: "gpt-6-luna-2026-09-01", + Usage: &openaischema.ResponseUsage{ + InputTokens: 100, + OutputTokens: 50, + TotalTokens: 150, + InputTokensDetails: openaischema.ResponseUsageInputTokensDetails{ + CachedTokens: 80, + CacheWriteTokens: 20, + }, + OutputTokensDetails: openaischema.ResponseUsageOutputTokensDetails{ReasoningTokens: 30}, + }, + } + responseJSON := `{"answers":[{"type":"choice","name":"route","choice":"billing"}],"model":"gpt-6-luna-2026-09-01","usage":{"input_tokens":100,"input_tokens_details":{"cached_tokens":80,"cache_write_tokens":20,"cache_creation_input_tokens":20},"output_tokens":50,"output_tokens_details":{"reasoning_tokens":30},"total_tokens":150}}` + metadataAttrs := []attribute.KeyValue{ + attribute.String(openinference.LLMModelName, "gpt-6-luna-2026-09-01"), + attribute.Int(openinference.LLMTokenCountPrompt, 100), + attribute.Int(openinference.LLMTokenCountCompletion, 50), + attribute.Int(openinference.LLMTokenCountTotal, 150), + attribute.Int(openinference.LLMTokenCountPromptCacheHit, 80), + attribute.Int(openinference.LLMTokenCountPromptCacheWrite, 20), + attribute.Int(openinference.LLMTokenCountCompletionReasoning, 30), + } + + for _, tc := range []struct { + name string + config *openinference.TraceConfig + expected string + mimeType bool + }{ + {name: "records response", config: &openinference.TraceConfig{}, expected: responseJSON, mimeType: true}, + {name: "redacts response", config: &openinference.TraceConfig{HideOutputs: true}, expected: openinference.RedactedValue}, + } { + t.Run(tc.name, func(t *testing.T) { + r := NewDecisionsRecorder(tc.config) + span := testotel.RecordWithSpan(t, func(span oteltrace.Span) bool { + r.RecordResponse(span, resp) + return false + }) + expected := append([]attribute.KeyValue{}, metadataAttrs...) + if tc.mimeType { + expected = append(expected, attribute.String(openinference.OutputMimeType, openinference.MimeTypeJSON)) + } + expected = append(expected, attribute.String(openinference.OutputValue, tc.expected)) + openinference.RequireAttributesEqual(t, expected, span.Attributes) + require.Equal(t, codes.Ok, span.Status.Code) + }) + } +} diff --git a/internal/tracing/otelgenai/backend_test.go b/internal/tracing/otelgenai/backend_test.go index ed8cb546bb..d2352676e4 100644 --- a/internal/tracing/otelgenai/backend_test.go +++ b/internal/tracing/otelgenai/backend_test.go @@ -134,6 +134,7 @@ func TestRecorder_RecordBackend_allEndpoints(t *testing.T) { "embeddings": NewEmbeddingsRecorder(cfg), "imageGeneration": NewImageGenerationRecorder(cfg), "responses": NewResponsesRecorder(cfg), + "decisions": NewDecisionsRecorder(cfg), "speech": NewSpeechRecorder(cfg), "transcription": NewTranscriptionRecorder(cfg), "translation": NewTranslationRecorder(cfg), diff --git a/internal/tracing/otelgenai/coverage_test.go b/internal/tracing/otelgenai/coverage_test.go index eb4313fc4b..ef87ed6127 100644 --- a/internal/tracing/otelgenai/coverage_test.go +++ b/internal/tracing/otelgenai/coverage_test.go @@ -97,6 +97,13 @@ func TestEndpointCoverage(t *testing.T) { actual: coverageOf(t, NewImageGenerationRecorder(cfg)), expected: coverage{}, }, + { + // Decisions is a custom operation; the response contributes the resolved + // model and token usage. + name: "decisions", + actual: coverageOf(t, NewDecisionsRecorder(cfg)), + expected: coverage{responseAttrs: true}, + }, { // The conventions define no speech attributes beyond the core set. name: "speech", diff --git a/internal/tracing/otelgenai/endpoints.go b/internal/tracing/otelgenai/endpoints.go index cc9d767f96..519d07d019 100644 --- a/internal/tracing/otelgenai/endpoints.go +++ b/internal/tracing/otelgenai/endpoints.go @@ -253,6 +253,30 @@ func NewResponsesRecorder(config *Config) tracingapi.ResponsesRecorder { } } +// NewDecisionsRecorder creates a recorder for the non-streaming Decisions API. +// Decisions has no standard GenAI operation name, so it uses a custom value. +func NewDecisionsRecorder(config *Config) tracingapi.DecisionsRecorder { + return &recorder[openai.DecisionRequest, openai.DecisionResponse, struct{}]{ + operation: OperationDecisions, + config: configOrEnv(config), + requestModel: func(r *openai.DecisionRequest) string { return r.Model }, + responseAttrs: decisionsResponseAttrs, + } +} + +func decisionsResponseAttrs(resp *openai.DecisionResponse) []attribute.KeyValue { + attrs := responseIdentityAttrs("", resp.Model) + if u := resp.Usage; u != nil { + attrs = append(attrs, usageAttrs(int(u.InputTokens), int(u.OutputTokens))...) + attrs = append(attrs, usageDetailAttrs( + int(u.InputTokensDetails.CachedTokens), + int(u.InputTokensDetails.CacheWriteTokensValue()), + int(u.OutputTokensDetails.ReasoningTokens), + )...) + } + return attrs +} + // NewSpeechRecorder creates a tracingapi.SpeechRecorder. func NewSpeechRecorder(config *Config) tracingapi.SpeechRecorder { return &recorder[openai.SpeechRequest, []byte, openai.SpeechStreamChunk]{ diff --git a/internal/tracing/otelgenai/endpoints_test.go b/internal/tracing/otelgenai/endpoints_test.go index 5ec7b9a4af..a1b815144e 100644 --- a/internal/tracing/otelgenai/endpoints_test.go +++ b/internal/tracing/otelgenai/endpoints_test.go @@ -63,6 +63,52 @@ func TestChatCompletionRecorder_RecordRequest(t *testing.T) { }, span.Attributes) } +func TestDecisionsRecorder_CoreAttributes(t *testing.T) { + r := NewDecisionsRecorder(NewConfig()) + spanName, opts := r.StartParams(&openai.DecisionRequest{Model: "gpt-6-luna"}, nil) + require.Equal(t, "decisions gpt-6-luna", spanName) + span := testotel.RecordNewSpan(t, spanName, opts...) + require.Equal(t, oteltrace.SpanKindClient, span.SpanKind) + + span = testotel.RecordWithSpan(t, func(span oteltrace.Span) bool { + r.RecordRequest(span, &openai.DecisionRequest{Model: "gpt-6-luna"}, nil) + return false + }) + testotel.RequireAttributesEqual(t, []attribute.KeyValue{ + attribute.String(OperationName, "decisions"), + attribute.String(RequestModel, "gpt-6-luna"), + }, span.Attributes) +} + +func TestDecisionsRecorder_RecordResponse(t *testing.T) { + r := NewDecisionsRecorder(NewConfig()) + span := testotel.RecordWithSpan(t, func(span oteltrace.Span) bool { + r.RecordResponse(span, &openai.DecisionResponse{ + Model: "gpt-6-luna-2026-09-01", + Usage: &openai.ResponseUsage{ + InputTokens: 100, + OutputTokens: 50, + InputTokensDetails: openai.ResponseUsageInputTokensDetails{ + CachedTokens: 80, + CacheWriteTokens: 20, + }, + OutputTokensDetails: openai.ResponseUsageOutputTokensDetails{ReasoningTokens: 30}, + }, + }) + return false + }) + + testotel.RequireAttributesEqual(t, []attribute.KeyValue{ + attribute.String(ResponseModel, "gpt-6-luna-2026-09-01"), + attribute.Int(UsageInputTokens, 100), + attribute.Int(UsageOutputTokens, 50), + attribute.Int(UsageCacheReadInputTokens, 80), + attribute.Int(UsageCacheCreationInputTokens, 20), + attribute.Int(UsageReasoningOutputTokens, 30), + }, span.Attributes) + require.Equal(t, codes.Ok, span.Status.Code) +} + // TestChatCompletionRecorder_RecordRequest_noContentByDefault pins that the raw // request body is not copied onto the span, which is the whole point of the // opt-in content default. diff --git a/internal/tracing/otelgenai/otelgenai.go b/internal/tracing/otelgenai/otelgenai.go index 0d9bd302ca..7c5ea31be9 100644 --- a/internal/tracing/otelgenai/otelgenai.go +++ b/internal/tracing/otelgenai/otelgenai.go @@ -90,6 +90,7 @@ const ( OperationTranslation Operation = "translation" OperationRerank Operation = "rerank" OperationSystemOne Operation = "systemone" + OperationDecisions Operation = "decisions" OperationTokenize Operation = "tokenize" ) diff --git a/internal/tracing/semconv.go b/internal/tracing/semconv.go index b2f61c4253..30ee9ddb3a 100644 --- a/internal/tracing/semconv.go +++ b/internal/tracing/semconv.go @@ -35,6 +35,7 @@ type recorderSet struct { embeddings tracingapi.EmbeddingsRecorder imageGeneration tracingapi.ImageGenerationRecorder responses tracingapi.ResponsesRecorder + decisions tracingapi.DecisionsRecorder speech tracingapi.SpeechRecorder transcription tracingapi.TranscriptionRecorder translation tracingapi.TranslationRecorder @@ -110,6 +111,7 @@ func newOpenInferenceRecorders() recorderSet { embeddings: openai.NewEmbeddingsRecorder(cfg), imageGeneration: openai.NewImageGenerationRecorder(cfg), responses: openai.NewResponsesRecorder(cfg), + decisions: openai.NewDecisionsRecorder(cfg), speech: openai.NewSpeechRecorder(cfg), transcription: openai.NewTranscriptionRecorder(cfg), translation: openai.NewTranslationRecorder(cfg), @@ -139,6 +141,7 @@ func newOTelGenAIRecorders() recorderSet { embeddings: otelgenai.NewEmbeddingsRecorder(cfg), imageGeneration: otelgenai.NewImageGenerationRecorder(cfg), responses: otelgenai.NewResponsesRecorder(cfg), + decisions: otelgenai.NewDecisionsRecorder(cfg), speech: otelgenai.NewSpeechRecorder(cfg), transcription: otelgenai.NewTranscriptionRecorder(cfg), translation: otelgenai.NewTranslationRecorder(cfg), diff --git a/internal/tracing/semconv_test.go b/internal/tracing/semconv_test.go index 4add0b3b0b..3dcfc909cc 100644 --- a/internal/tracing/semconv_test.go +++ b/internal/tracing/semconv_test.go @@ -158,6 +158,7 @@ func requireAllRecordersSet(t *testing.T, r *recorderSet) { require.NotNil(t, r.embeddings, "embeddings") require.NotNil(t, r.imageGeneration, "imageGeneration") require.NotNil(t, r.responses, "responses") + require.NotNil(t, r.decisions, "decisions") require.NotNil(t, r.speech, "speech") require.NotNil(t, r.transcription, "transcription") require.NotNil(t, r.translation, "translation") diff --git a/internal/tracing/span.go b/internal/tracing/span.go index f727fa5732..4ac15a348f 100644 --- a/internal/tracing/span.go +++ b/internal/tracing/span.go @@ -61,6 +61,7 @@ type ( embeddingsSpan = span[openai.EmbeddingResponse, struct{}] imageGenerationSpan = span[openai.ImageGenerationResponse, struct{}] responsesSpan = span[openai.Response, openai.ResponseStreamEventUnion] + decisionsSpan = span[openai.DecisionResponse, struct{}] speechSpan = span[[]byte, openai.SpeechStreamChunk] transcriptionSpan = span[openai.TranscriptionResponse, openai.TranscriptionStreamEvent] translationSpan = span[openai.TranslationResponse, struct{}] diff --git a/internal/tracing/tracer.go b/internal/tracing/tracer.go index 9c83a6bd12..d14d2c825a 100644 --- a/internal/tracing/tracer.go +++ b/internal/tracing/tracer.go @@ -39,6 +39,7 @@ var ( _ tracingapi.CompletionTracer = (*completionTracer)(nil) _ tracingapi.ImageGenerationTracer = (*imageGenerationTracer)(nil) _ tracingapi.ResponsesTracer = (*responsesTracer)(nil) + _ tracingapi.DecisionsTracer = (*decisionsTracer)(nil) _ tracingapi.SpeechTracer = (*speechTracer)(nil) _ tracingapi.TranscriptionTracer = (*transcriptionTracer)(nil) _ tracingapi.TranslationTracer = (*translationTracer)(nil) @@ -55,6 +56,7 @@ type ( completionTracer = requestTracerImpl[openai.CompletionRequest, openai.CompletionResponse, openai.CompletionResponse] imageGenerationTracer = requestTracerImpl[openai.ImageGenerationRequest, openai.ImageGenerationResponse, struct{}] responsesTracer = requestTracerImpl[openai.ResponseRequest, openai.Response, openai.ResponseStreamEventUnion] + decisionsTracer = requestTracerImpl[openai.DecisionRequest, openai.DecisionResponse, struct{}] speechTracer = requestTracerImpl[openai.SpeechRequest, []byte, openai.SpeechStreamChunk] transcriptionTracer = requestTracerImpl[openai.TranscriptionRequest, openai.TranscriptionResponse, openai.TranscriptionStreamEvent] translationTracer = requestTracerImpl[openai.TranslationRequest, openai.TranslationResponse, struct{}] @@ -179,6 +181,18 @@ func newResponsesTracer(tracer trace.Tracer, propagator propagation.TextMapPropa ) } +func newDecisionsTracer(tracer trace.Tracer, propagator propagation.TextMapPropagator, recorder tracingapi.DecisionsRecorder, headerAttributes map[string]string) tracingapi.DecisionsTracer { + return newRequestTracer( + tracer, + propagator, + recorder, + headerAttributes, + func(span trace.Span, recorder tracingapi.DecisionsRecorder) tracingapi.DecisionsSpan { + return &decisionsSpan{span: span, recorder: recorder} + }, + ) +} + func newSpeechTracer(tracer trace.Tracer, propagator propagation.TextMapPropagator, recorder tracingapi.SpeechRecorder, headerAttributes map[string]string) tracingapi.SpeechTracer { return newRequestTracer( tracer, diff --git a/internal/tracing/tracing.go b/internal/tracing/tracing.go index f09267c9e4..8c0f468f5f 100644 --- a/internal/tracing/tracing.go +++ b/internal/tracing/tracing.go @@ -29,6 +29,7 @@ type tracingImpl struct { imageGenerationTracer tracingapi.ImageGenerationTracer embeddingsTracer tracingapi.EmbeddingsTracer responsesTracer tracingapi.ResponsesTracer + decisionsTracer tracingapi.DecisionsTracer speechTracer tracingapi.SpeechTracer transcriptionTracer tracingapi.TranscriptionTracer translationTracer tracingapi.TranslationTracer @@ -68,6 +69,11 @@ func (t *tracingImpl) ResponsesTracer() tracingapi.ResponsesTracer { return t.responsesTracer } +// DecisionsTracer implements the same method as documented on tracingapi.Tracing. +func (t *tracingImpl) DecisionsTracer() tracingapi.DecisionsTracer { + return t.decisionsTracer +} + // SpeechTracer implements the same method as documented on tracingapi.Tracing. func (t *tracingImpl) SpeechTracer() tracingapi.SpeechTracer { return t.speechTracer @@ -268,6 +274,12 @@ func NewTracingFromEnv(ctx context.Context, stdout io.Writer, headerAttributeMap recorders.responses, headerAttrs, ), + decisionsTracer: newDecisionsTracer( + tracer, + propagator, + recorders.decisions, + headerAttrs, + ), speechTracer: newSpeechTracer( tracer, propagator, diff --git a/internal/tracing/tracing_test.go b/internal/tracing/tracing_test.go index 1c8d9091ec..019a3983db 100644 --- a/internal/tracing/tracing_test.go +++ b/internal/tracing/tracing_test.go @@ -876,6 +876,7 @@ func TestNewTracingFromEnv_everyTracerWired(t *testing.T) { require.NotNil(t, tracing.EmbeddingsTracer(), "EmbeddingsTracer") require.NotNil(t, tracing.ImageGenerationTracer(), "ImageGenerationTracer") require.NotNil(t, tracing.ResponsesTracer(), "ResponsesTracer") + require.NotNil(t, tracing.DecisionsTracer(), "DecisionsTracer") require.NotNil(t, tracing.SpeechTracer(), "SpeechTracer") require.NotNil(t, tracing.TranscriptionTracer(), "TranscriptionTracer") require.NotNil(t, tracing.TranslationTracer(), "TranslationTracer") diff --git a/internal/tracing/tracingapi/api.go b/internal/tracing/tracingapi/api.go index adcad22d52..d4713af1e8 100644 --- a/internal/tracing/tracingapi/api.go +++ b/internal/tracing/tracingapi/api.go @@ -34,6 +34,8 @@ type ( EmbeddingsTracer() EmbeddingsTracer // ResponsesTracer creates spans for OpenAI responses requests on /v1/responses endpoint. ResponsesTracer() ResponsesTracer + // DecisionsTracer creates spans for OpenAI Decisions requests on /v1/decisions. + DecisionsTracer() DecisionsTracer // SpeechTracer creates spans for OpenAI speech synthesis requests on /v1/audio/speech endpoint. SpeechTracer() SpeechTracer // TranscriptionTracer creates spans for OpenAI audio transcription requests on /v1/audio/transcriptions endpoint. @@ -81,6 +83,8 @@ type ( ImageGenerationTracer = RequestTracer[openai.ImageGenerationRequest, openai.ImageGenerationResponse, struct{}] // ResponsesTracer creates spans for OpenAI responses requests. ResponsesTracer = RequestTracer[openai.ResponseRequest, openai.Response, openai.ResponseStreamEventUnion] + // DecisionsTracer creates spans for OpenAI Decisions requests. + DecisionsTracer = RequestTracer[openai.DecisionRequest, openai.DecisionResponse, struct{}] // SpeechTracer creates spans for OpenAI speech synthesis requests. SpeechTracer = RequestTracer[openai.SpeechRequest, []byte, openai.SpeechStreamChunk] // TranscriptionTracer creates spans for OpenAI audio transcription requests. @@ -129,6 +133,8 @@ type ( ImageGenerationSpan = Span[openai.ImageGenerationResponse, struct{}] // ResponsesSpan represents an OpenAI responses request span. ResponsesSpan = Span[openai.Response, openai.ResponseStreamEventUnion] + // DecisionsSpan represents an OpenAI Decisions request span. + DecisionsSpan = Span[openai.DecisionResponse, struct{}] // SpeechSpan represents an OpenAI speech synthesis request span. SpeechSpan = Span[[]byte, openai.SpeechStreamChunk] // TranscriptionSpan represents an OpenAI audio transcription request span. @@ -186,6 +192,8 @@ type ( EmbeddingsRecorder = SpanRecorder[openai.EmbeddingRequest, openai.EmbeddingResponse, struct{}] // ResponsesRecorder records attributes to a span according to a semantic convention. ResponsesRecorder = SpanRecorder[openai.ResponseRequest, openai.Response, openai.ResponseStreamEventUnion] + // DecisionsRecorder records attributes to a span according to a semantic convention. + DecisionsRecorder = SpanRecorder[openai.DecisionRequest, openai.DecisionResponse, struct{}] // SpeechRecorder records attributes to a span according to a semantic convention. SpeechRecorder = SpanRecorder[openai.SpeechRequest, []byte, openai.SpeechStreamChunk] // TranscriptionRecorder records attributes to a span according to a semantic convention. @@ -272,6 +280,11 @@ func (NoopTracing) ResponsesTracer() ResponsesTracer { return NoopResponsesTracer{} } +// DecisionsTracer implements Tracing.DecisionsTracer. +func (NoopTracing) DecisionsTracer() DecisionsTracer { + return NoopDecisionsTracer{} +} + // SpeechTracer implements Tracing.SpeechTracer. func (NoopTracing) SpeechTracer() SpeechTracer { return NoopSpeechTracer{} @@ -334,6 +347,8 @@ type ( NoopImageGenerationTracer = NoopTracer[openai.ImageGenerationRequest, openai.ImageGenerationResponse, struct{}] // NoopResponsesTracer implements ResponsesTracer. NoopResponsesTracer = NoopTracer[openai.ResponseRequest, openai.Response, openai.ResponseStreamEventUnion] + // NoopDecisionsTracer implements DecisionsTracer. + NoopDecisionsTracer = NoopTracer[openai.DecisionRequest, openai.DecisionResponse, struct{}] // NoopSpeechTracer implements SpeechTracer. NoopSpeechTracer = NoopTracer[openai.SpeechRequest, []byte, openai.SpeechStreamChunk] // NoopTranscriptionTracer implements TranscriptionTracer. diff --git a/internal/translator/openai_decisions.go b/internal/translator/openai_decisions.go new file mode 100644 index 0000000000..4c0cd12ca4 --- /dev/null +++ b/internal/translator/openai_decisions.go @@ -0,0 +1,93 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package translator + +import ( + "fmt" + "io" + "path" + "strconv" + + "github.com/tidwall/sjson" + + "github.com/envoyproxy/ai-gateway/internal/apischema/openai" + "github.com/envoyproxy/ai-gateway/internal/internalapi" + "github.com/envoyproxy/ai-gateway/internal/json" + "github.com/envoyproxy/ai-gateway/internal/metrics" + "github.com/envoyproxy/ai-gateway/internal/tracing/tracingapi" +) + +// NewDecisionsOpenAIToOpenAITranslator creates an OpenAI Decisions passthrough +// translator with optional path prefixing and model virtualization. +func NewDecisionsOpenAIToOpenAITranslator(prefix string, modelNameOverride internalapi.ModelNameOverride) OpenAIDecisionsTranslator { + return &openAIToOpenAITranslatorV1Decisions{ + modelNameOverride: modelNameOverride, + path: path.Join("/", prefix, "decisions"), + } +} + +type openAIToOpenAITranslatorV1Decisions struct { + modelNameOverride internalapi.ModelNameOverride + path string + requestModel internalapi.RequestModel +} + +// RequestBody preserves the Decisions wire format and changes only the model +// when a route configured model virtualization. +func (o *openAIToOpenAITranslatorV1Decisions) RequestBody(original []byte, req *openai.DecisionRequest, forceBodyMutation bool) ( + newHeaders []internalapi.Header, newBody []byte, err error, +) { + o.requestModel = req.Model + if o.modelNameOverride != "" { + newBody, err = sjson.SetBytesOptions(original, "model", o.modelNameOverride, sjsonOptions) + if err != nil { + return nil, nil, fmt.Errorf("failed to set model: %w", err) + } + o.requestModel = o.modelNameOverride + } + + newHeaders = []internalapi.Header{{pathHeaderName, o.path}} + newBody = forceOriginalBodyIfEmpty(forceBodyMutation, newBody, original) + if len(newBody) > 0 { + newHeaders = append(newHeaders, internalapi.Header{contentLengthHeaderName, strconv.Itoa(len(newBody))}) + } + return +} + +func (*openAIToOpenAITranslatorV1Decisions) ResponseHeaders(map[string]string) ([]internalapi.Header, error) { + return nil, nil +} + +// ResponseBody decodes the response for tracing and token accounting but leaves +// the upstream bytes untouched. +func (o *openAIToOpenAITranslatorV1Decisions) ResponseBody(_ map[string]string, body io.Reader, _ bool, span tracingapi.DecisionsSpan) ( + newHeaders []internalapi.Header, newBody []byte, tokenUsage metrics.TokenUsage, responseModel internalapi.ResponseModel, err error, +) { + var resp openai.DecisionResponse + if err := json.NewDecoder(body).Decode(&resp); err != nil { + return nil, nil, tokenUsage, o.requestModel, fmt.Errorf("failed to unmarshal body: %w", err) + } + if span != nil { + span.RecordResponse(&resp) + } + responseModel = resp.Model + if responseModel == "" { + responseModel = o.requestModel + } + if resp.Usage != nil { + tokenUsage.SetInputTokens(uint32(resp.Usage.InputTokens)) // #nosec G115 + tokenUsage.SetOutputTokens(uint32(resp.Usage.OutputTokens)) // #nosec G115 + tokenUsage.SetTotalTokens(uint32(resp.Usage.TotalTokens)) // #nosec G115 + tokenUsage.SetCachedInputTokens(uint32(resp.Usage.InputTokensDetails.CachedTokens)) // #nosec G115 + tokenUsage.SetCacheCreationInputTokens(uint32(resp.Usage.InputTokensDetails.CacheWriteTokensValue())) // #nosec G115 + tokenUsage.SetReasoningTokens(uint32(resp.Usage.OutputTokensDetails.ReasoningTokens)) // #nosec G115 + } + return nil, nil, tokenUsage, responseModel, nil +} + +func (*openAIToOpenAITranslatorV1Decisions) ResponseError(respHeaders map[string]string, body io.Reader) ([]internalapi.Header, []byte, error) { + return convertErrorOpenAIToOpenAIError(respHeaders, body) +} diff --git a/internal/translator/openai_decisions_test.go b/internal/translator/openai_decisions_test.go new file mode 100644 index 0000000000..76fa844a08 --- /dev/null +++ b/internal/translator/openai_decisions_test.go @@ -0,0 +1,125 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package translator + +import ( + "bytes" + "strconv" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/envoyproxy/ai-gateway/internal/apischema/openai" + "github.com/envoyproxy/ai-gateway/internal/json" +) + +func TestDecisionsOpenAIToOpenAITranslator_RequestBody(t *testing.T) { + t.Run("passthrough", func(t *testing.T) { + tr := NewDecisionsOpenAIToOpenAITranslator("v1", "").(*openAIToOpenAITranslatorV1Decisions) + original := []byte(`{"model":"gpt-6-luna","input":"hello","questions":[]}`) + headers, body, err := tr.RequestBody(original, &openai.DecisionRequest{Model: "gpt-6-luna"}, false) + require.NoError(t, err) + require.Equal(t, []string{":path", "/v1/decisions"}, []string{headers[0].Key(), headers[0].Value()}) + require.Nil(t, body) + require.Equal(t, "gpt-6-luna", tr.requestModel) + }) + + t.Run("custom prefix and model override", func(t *testing.T) { + tr := NewDecisionsOpenAIToOpenAITranslator("custom/v1", "provider-luna").(*openAIToOpenAITranslatorV1Decisions) + original := []byte(`{"model":"gpt-6-luna","input":"hello","questions":[]}`) + headers, body, err := tr.RequestBody(original, &openai.DecisionRequest{Model: "gpt-6-luna"}, false) + require.NoError(t, err) + require.Equal(t, "/custom/v1/decisions", headers[0].Value()) + require.Equal(t, contentLengthHeaderName, headers[1].Key()) + require.Equal(t, strconv.Itoa(len(body)), headers[1].Value()) + require.JSONEq(t, `{"model":"provider-luna","input":"hello","questions":[]}`, string(body)) + require.Equal(t, "provider-luna", tr.requestModel) + }) + + t.Run("forced body mutation", func(t *testing.T) { + tr := NewDecisionsOpenAIToOpenAITranslator("v1", "") + original := []byte(`{"model":"gpt-6-luna","input":"hello","questions":[]}`) + headers, body, err := tr.RequestBody(original, &openai.DecisionRequest{Model: "gpt-6-luna"}, true) + require.NoError(t, err) + require.Equal(t, original, body) + require.Equal(t, strconv.Itoa(len(body)), headers[1].Value()) + }) +} + +func TestDecisionsOpenAIToOpenAITranslator_ResponseBody(t *testing.T) { + tr := NewDecisionsOpenAIToOpenAITranslator("v1", "provider-luna").(*openAIToOpenAITranslatorV1Decisions) + _, _, err := tr.RequestBody([]byte(`{"model":"gpt-6-luna"}`), &openai.DecisionRequest{Model: "gpt-6-luna"}, false) + require.NoError(t, err) + + response := []byte(`{ + "model": "gpt-6-luna-2026-09-01", + "answers": [ + {"type":"predicate","name":"relevant","probability":0}, + {"type":"choice","name":"approved","choice":true,"probabilities":[{"value":true,"probability":1}],"confidence":1}, + {"type":"score","name":"severity","score":1.1,"probabilities":[{"value":1,"label":"medium","probability":0.9}],"confidence":0.8}, + {"type":"refusal","name":"unsafe"} + ], + "usage": { + "input_tokens": 20, + "input_tokens_details": {"cache_write_tokens": 3, "cached_tokens": 4}, + "output_tokens": 5, + "output_tokens_details": {"reasoning_tokens": 2}, + "total_tokens": 25 + } + }`) + + _, body, usage, responseModel, err := tr.ResponseBody(nil, bytes.NewReader(response), true, nil) + require.NoError(t, err) + require.Nil(t, body) + require.Equal(t, "gpt-6-luna-2026-09-01", responseModel) + inputTokens, hasInput := usage.InputTokens() + require.True(t, hasInput) + require.Equal(t, uint32(20), inputTokens) + outputTokens, hasOutput := usage.OutputTokens() + require.True(t, hasOutput) + require.Equal(t, uint32(5), outputTokens) + totalTokens, hasTotal := usage.TotalTokens() + require.True(t, hasTotal) + require.Equal(t, uint32(25), totalTokens) + cachedTokens, hasCached := usage.CachedInputTokens() + require.True(t, hasCached) + require.Equal(t, uint32(4), cachedTokens) + cacheWriteTokens, hasCacheWrite := usage.CacheCreationInputTokens() + require.True(t, hasCacheWrite) + require.Equal(t, uint32(3), cacheWriteTokens) + reasoningTokens, hasReasoning := usage.ReasoningTokens() + require.True(t, hasReasoning) + require.Equal(t, uint32(2), reasoningTokens) + + var decoded openai.DecisionResponse + require.NoError(t, json.Unmarshal(response, &decoded)) + require.Len(t, decoded.Answers, 4) + require.NotNil(t, decoded.Answers[0].Probability) + require.Zero(t, *decoded.Answers[0].Probability) + require.JSONEq(t, `true`, string(decoded.Answers[1].Choice)) + require.JSONEq(t, `1`, string(decoded.Answers[2].Probabilities[0].Value)) +} + +func TestDecisionsOpenAIToOpenAITranslator_ResponseBody_FallsBackToRequestModelWithoutUsage(t *testing.T) { + tr := NewDecisionsOpenAIToOpenAITranslator("v1", "provider-luna").(*openAIToOpenAITranslatorV1Decisions) + _, _, err := tr.RequestBody([]byte(`{"model":"gpt-6-luna"}`), &openai.DecisionRequest{Model: "gpt-6-luna"}, false) + require.NoError(t, err) + + _, _, usage, responseModel, err := tr.ResponseBody(nil, bytes.NewReader([]byte(`{"answers":[]}`)), true, nil) + require.NoError(t, err) + require.Equal(t, "provider-luna", responseModel) + _, hasInput := usage.InputTokens() + require.False(t, hasInput) +} + +func TestDecisionsOpenAIToOpenAITranslator_ResponseError(t *testing.T) { + tr := NewDecisionsOpenAIToOpenAITranslator("v1", "") + body := []byte(`{"error":{"message":"bad request","type":"invalid_request_error"}}`) + headers, mutated, err := tr.ResponseError(map[string]string{"content-type": "application/json", ":status": "400"}, bytes.NewReader(body)) + require.NoError(t, err) + require.Nil(t, headers) + require.Nil(t, mutated) +} diff --git a/internal/translator/translator.go b/internal/translator/translator.go index 5555c1f95c..4cfb678a6b 100644 --- a/internal/translator/translator.go +++ b/internal/translator/translator.go @@ -145,6 +145,8 @@ type ( OpenAIImageGenerationTranslator = Translator[openai.ImageGenerationRequest, tracingapi.ImageGenerationSpan] // OpenAIResponsesTranslator translates the OpenAI's /responses endpoint. OpenAIResponsesTranslator = Translator[openai.ResponseRequest, tracingapi.ResponsesSpan] + // OpenAIDecisionsTranslator translates the OpenAI /v1/decisions endpoint. + OpenAIDecisionsTranslator = Translator[openai.DecisionRequest, tracingapi.DecisionsSpan] // OpenAISpeechTranslator translates the OpenAI's /v1/audio/speech endpoint. OpenAISpeechTranslator = Translator[openai.SpeechRequest, tracingapi.SpeechSpan] // OpenAIAudioTranscriptionTranslator translates the OpenAI's /v1/audio/transcriptions endpoint. diff --git a/site/docs/capabilities/llm-integrations/supported-endpoints.md b/site/docs/capabilities/llm-integrations/supported-endpoints.md index ed83453c1c..b4174f839a 100644 --- a/site/docs/capabilities/llm-integrations/supported-endpoints.md +++ b/site/docs/capabilities/llm-integrations/supported-endpoints.md @@ -351,6 +351,50 @@ curl -H "Content-Type: application/json" \ $GATEWAY_URL/v1/responses ``` +### Decisions + +**Endpoint:** `POST /v1/decisions` + +**Status:** ✅ Supported for OpenAI backends + +**Description:** Evaluate shared text or image evidence against typed questions and return probabilities, fixed choices, or rubric scores. The request and response use the native OpenAI Decisions API format. + +**Features:** + +- ✅ `predicate`, `choice`, and `score` questions, including multiple independent questions in one request +- ✅ `refusal` answers when the model declines an individual question +- ✅ Text input and user messages containing text and inline base64 images +- ✅ Model selection via request body or `x-ai-eg-model` header +- ✅ Model name virtualization, provider fallback, and load balancing +- ✅ Token usage and model metadata from the upstream response +- ❌ Streaming (not documented by the OpenAI Decisions API) +- ❌ Hosted image URLs and file IDs (not supported by the OpenAI Decisions API) + +**Supported Providers:** + +- OpenAI + +**Example:** + +```bash +curl -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-6-luna", + "input": "I was charged twice for my order.", + "questions": [{ + "type": "choice", + "name": "department", + "instructions": "Which department should handle this complaint?", + "choices": [ + {"value": "billing", "description": "Payments, invoices, and refunds."}, + {"value": "technical", "description": "Problems using the product."}, + {"value": "other", "description": "Requests outside these categories."} + ] + }] + }' \ + $GATEWAY_URL/v1/decisions +``` + ### Responses Input Tokens **Endpoint:** `POST /v1/responses/input_tokens` diff --git a/tests/data-plane/testupstream_test.go b/tests/data-plane/testupstream_test.go index 32596a2fe0..f784a110e8 100644 --- a/tests/data-plane/testupstream_test.go +++ b/tests/data-plane/testupstream_test.go @@ -1297,6 +1297,17 @@ data: {"type":"message_stop","amazon-bedrock-invocationMetrics":{"inputTokenCoun expStatus: http.StatusOK, expResponseBody: `{"id":"resp_67cc","object":"response","created_at":1741476542,"status":"completed","model":"something","output":[{"type":"message","id":"msg_67c","status":"completed","role":"assistant","content":[{"type":"output_text","text":"This is a test."}]}],"parallel_tool_calls":true,"store":true,"temperature":1,"text":{"format":{"type":"text"}},"tool_choice":"auto","top_p":1,"truncation":"disabled","usage":{"input_tokens":16,"input_tokens_details":{"cached_tokens":5},"output_tokens":6,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":22}}`, }, + { + name: "openai - /v1/decisions", + backend: "openai", + path: "/v1/decisions", + method: http.MethodPost, + requestBody: `{"model":"gpt-6-luna","input":"I was charged twice.","questions":[{"type":"choice","name":"department","instructions":"Route this request.","choices":[{"value":"billing","description":"Billing requests."},{"value":"other","description":"Other requests."}]}]}`, + expPath: "/v1/decisions", + responseBody: `{"answers":[{"type":"choice","name":"department","choice":"billing","probabilities":[{"value":"billing","probability":0.95},{"value":"other","probability":0.05}],"confidence":0.95}],"model":"gpt-6-luna","usage":{"input_tokens":24,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":24}}`, + expStatus: http.StatusOK, + expResponseBody: `{"answers":[{"type":"choice","name":"department","choice":"billing","probabilities":[{"value":"billing","probability":0.95},{"value":"other","probability":0.05}],"confidence":0.95}],"model":"gpt-6-luna","usage":{"input_tokens":24,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":24}}`, + }, { name: "azure-openai - /v1/responses", backend: "azure-openai", From 4ab304dd14f3abb19a782dcb29d174aa644b7ba3 Mon Sep 17 00:00:00 2001 From: yxia216 Date: Thu, 8 Oct 2026 11:51:28 -0400 Subject: [PATCH 2/2] improve-code-stype Signed-off-by: yxia216 --- internal/apischema/openai/decisions.go | 22 ++++++++++++++----- internal/endpointspec/endpointspec.go | 1 + internal/endpointspec/endpointspec_test.go | 4 +++- .../llm-integrations/supported-endpoints.md | 4 ++-- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/internal/apischema/openai/decisions.go b/internal/apischema/openai/decisions.go index 9ef997e198..20c494a82b 100644 --- a/internal/apischema/openai/decisions.go +++ b/internal/apischema/openai/decisions.go @@ -10,34 +10,42 @@ import "github.com/envoyproxy/ai-gateway/internal/json" // DecisionRequest represents the request body for POST /v1/decisions. // Input is kept as raw JSON because the API accepts either a string or an array // of user messages containing text and inline base64 images. +// Docs: https://developers.openai.com/api/reference/resources/decisions/methods/create type DecisionRequest struct { Model string `json:"model"` Input json.RawMessage `json:"input"` Questions []DecisionQuestion `json:"questions"` + // SafetyIdentifier is an opaque caller-provided end-user identifier. + SafetyIdentifier string `json:"safety_identifier,omitempty"` } // DecisionQuestion is a predicate, choice, or score question. +// Docs: https://developers.openai.com/api/docs/guides/decisions +// SDK: https://github.com/openai/openai-go/blob/v3.73.0/decision.go type DecisionQuestion struct { Type string `json:"type"` - Name string `json:"name"` + Name string `json:"name,omitempty"` Instructions string `json:"instructions"` Choices []DecisionChoiceOption `json:"choices,omitempty"` Levels []DecisionScoreLevel `json:"levels,omitempty"` } -// DecisionChoiceOption is one allowed result for a choice question. +// DecisionChoiceOption is one allowed result for a choice question. Value is raw because choice values are typed as either strings or booleans. +// SDK: https://github.com/openai/openai-go/blob/v3.73.0/decision.go type DecisionChoiceOption struct { Value json.RawMessage `json:"value"` - Description string `json:"description"` + Description string `json:"description,omitempty"` } // DecisionScoreLevel is one ordered level for a score question. +// Docs: https://developers.openai.com/api/docs/guides/decisions type DecisionScoreLevel struct { Label string `json:"label"` - Description string `json:"description"` + Description string `json:"description,omitempty"` } // DecisionResponse represents a response from POST /v1/decisions. +// Docs: https://developers.openai.com/api/reference/resources/decisions/methods/create type DecisionResponse struct { Answers []DecisionAnswer `json:"answers"` Model string `json:"model,omitempty"` @@ -47,9 +55,11 @@ type DecisionResponse struct { // DecisionAnswer is a predicate, choice, score, or refusal answer. Pointer // numeric fields preserve valid zero values while omitting fields belonging to // other variants. +// Docs: https://developers.openai.com/api/reference/resources/decisions/methods/create +// Docs: https://developers.openai.com/api/docs/guides/decisions type DecisionAnswer struct { Type string `json:"type"` - Name string `json:"name"` + Name string `json:"name,omitempty"` Probability *float64 `json:"probability,omitempty"` Choice json.RawMessage `json:"choice,omitempty"` Probabilities []DecisionProbability `json:"probabilities,omitempty"` @@ -60,6 +70,8 @@ type DecisionAnswer struct { // DecisionProbability is one entry in a choice or score probability // distribution. Value is raw because choices use strings or booleans, while // score levels use numeric indices. +// Docs: https://developers.openai.com/api/docs/guides/decisions +// SDK: https://github.com/openai/openai-go/blob/v3.73.0/decision.go type DecisionProbability struct { Value json.RawMessage `json:"value"` Label string `json:"label,omitempty"` diff --git a/internal/endpointspec/endpointspec.go b/internal/endpointspec/endpointspec.go index 790cd2599b..5010ebb45b 100644 --- a/internal/endpointspec/endpointspec.go +++ b/internal/endpointspec/endpointspec.go @@ -445,6 +445,7 @@ func (DecisionsEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, func (DecisionsEndpointSpec) RedactSensitiveInfoFromRequest(req *openai.DecisionRequest) (*openai.DecisionRequest, error) { redacted := *req redacted.Input = redactRawJSON(req.Input) + redacted.SafetyIdentifier = redaction.RedactString(req.SafetyIdentifier) redacted.Questions = make([]openai.DecisionQuestion, len(req.Questions)) for i := range req.Questions { q := req.Questions[i] diff --git a/internal/endpointspec/endpointspec_test.go b/internal/endpointspec/endpointspec_test.go index 76b9368bdd..6353e66790 100644 --- a/internal/endpointspec/endpointspec_test.go +++ b/internal/endpointspec/endpointspec_test.go @@ -556,7 +556,7 @@ func TestDecisionsEndpointSpec(t *testing.T) { } func TestDecisionsEndpointSpec_RedactSensitiveInfoFromRequest(t *testing.T) { - body := []byte(`{"model":"gpt-6-luna","input":"customer complaint","questions":[{"type":"choice","name":"department","instructions":"route this complaint","choices":[{"value":"billing","description":"payment issues"}]},{"type":"score","name":"severity","instructions":"score it","levels":[{"label":"critical","description":"no workaround"}]}]}`) + body := []byte(`{"model":"gpt-6-luna","input":"customer complaint","safety_identifier":"user-hash-123","questions":[{"type":"choice","name":"department","instructions":"route this complaint","choices":[{"value":"billing","description":"payment issues"}]},{"type":"score","name":"severity","instructions":"score it","levels":[{"label":"critical","description":"no workaround"}]}]}`) _, req, _, _, err := DecisionsEndpointSpec{}.ParseBody(body, false) require.NoError(t, err) redacted, err := DecisionsEndpointSpec{}.RedactSensitiveInfoFromRequest(req) @@ -569,6 +569,8 @@ func TestDecisionsEndpointSpec_RedactSensitiveInfoFromRequest(t *testing.T) { require.NotContains(t, text, "customer complaint") require.NotContains(t, text, "payment issues") require.NotContains(t, text, "critical") + require.NotContains(t, text, "user-hash-123") + require.Contains(t, text, `"safety_identifier":"[REDACTED LENGTH=`) require.Contains(t, text, `"model":"gpt-6-luna"`) require.Contains(t, text, `"name":"department"`) } diff --git a/site/docs/capabilities/llm-integrations/supported-endpoints.md b/site/docs/capabilities/llm-integrations/supported-endpoints.md index b4174f839a..a76cc55e46 100644 --- a/site/docs/capabilities/llm-integrations/supported-endpoints.md +++ b/site/docs/capabilities/llm-integrations/supported-endpoints.md @@ -355,9 +355,9 @@ curl -H "Content-Type: application/json" \ **Endpoint:** `POST /v1/decisions` -**Status:** ✅ Supported for OpenAI backends +**Status:** ✅ Supported for OpenAI backends (beta) -**Description:** Evaluate shared text or image evidence against typed questions and return probabilities, fixed choices, or rubric scores. The request and response use the native OpenAI Decisions API format. +**Description:** Evaluate shared text or image evidence against typed questions and return probabilities, fixed choices, or rubric scores. The request and response use the native OpenAI Decisions API format. OpenAI offers this API as a beta, so its request and response formats may change. **Features:**