From afae761d79871866289ecb7e5ba6fb14aac31027 Mon Sep 17 00:00:00 2001 From: doxuta Date: Tue, 27 Oct 2026 14:39:12 +0700 Subject: [PATCH 1/7] feat(ai): stream completions with usage and fold the stream into one answer (#30 item 8) The gateway asks for stream + stream_options.include_usage and folds the event stream back into the ChatResponse the agents, the recorder and replay already read: text and refusal text joined, tool-call arguments joined by index, finish/stop reasons and the last usage kept. A gateway that ignores stream and answers one JSON body is read as before. An error event, a stream cut before it finished or a non-JSON chunk is a 502; a stall is a 504. Co-Authored-By: Claude Opus 5.5 --- nexus-platform/internal/ai/gateway.go | 194 +++++++++-- .../internal/ai/gateway_stream_test.go | 321 ++++++++++++++++++ nexus-platform/internal/ai/gateway_test.go | 6 +- .../internal/handlers/ai_refusal_test.go | 17 +- 4 files changed, 501 insertions(+), 37 deletions(-) create mode 100644 nexus-platform/internal/ai/gateway_stream_test.go diff --git a/nexus-platform/internal/ai/gateway.go b/nexus-platform/internal/ai/gateway.go index a991b17..588a9f6 100644 --- a/nexus-platform/internal/ai/gateway.go +++ b/nexus-platform/internal/ai/gateway.go @@ -85,14 +85,21 @@ type ToolFunction struct { Parameters map[string]any `json:"parameters"` } -// ChatRequest is the POST /chat/completions body. Stream is always false (the loop needs the whole -// response before deciding whether to run tools). MaxTokens bounds a single completion. +// ChatRequest is the POST /chat/completions body. MaxTokens bounds a single completion. Callers leave Stream +// and StreamOptions unset: Gateway.Complete sets them on the wire (it streams to get the whole usage, then +// folds the stream back into one ChatResponse), so a cassette records the request as the agent built it. type ChatRequest struct { - Model string `json:"model"` - Messages []ChatMessage `json:"messages"` - Tools []Tool `json:"tools,omitempty"` - Stream bool `json:"stream"` - MaxTokens int `json:"max_tokens,omitempty"` + Model string `json:"model"` + Messages []ChatMessage `json:"messages"` + Tools []Tool `json:"tools,omitempty"` + Stream bool `json:"stream"` + StreamOptions *StreamOptions `json:"stream_options,omitempty"` + MaxTokens int `json:"max_tokens,omitempty"` +} + +// StreamOptions asks a streamed completion to end with the call's usage (OpenAI's include_usage). +type StreamOptions struct { + IncludeUsage bool `json:"include_usage"` } // ChatResponse is the subset of the OpenAI-compatible response the loop consumes. StopReason is Claude's @@ -146,9 +153,11 @@ type Usage struct { // - Both at once (an OpenAI-compatible proxy in front of Anthropic): prompt_tokens is taken to include // the cache fields, as OpenAI's does, unless it is smaller than them and so cannot. // -// A field that is absent counts 0. A gateway that sends no cache figures leaves both at 0 and its -// prompt_tokens a floor: 9router's non-streaming answer for a Claude model (the dev gateway) drops -// them. ponytail: DeepSeek's prompt_cache_hit_tokens and other vendor +// A field that is absent counts 0, so a gateway that sends no cache figures leaves both at 0. The dev +// gateway (9router 0.5.99, a Claude model) is one: its OpenAI-shape usage keeps only prompt_tokens, +// completion_tokens and total_tokens, and it adds 2,000 to prompt_tokens. Streamed (what Gateway.Complete +// asks for), prompt_tokens is the whole input plus that 2,000; not streamed, it was the uncached +// remainder plus 2,000, a floor. ponytail: DeepSeek's prompt_cache_hit_tokens and other vendor // spellings are not read; add one here when a gateway sends it. func (u *Usage) UnmarshalJSON(b []byte) error { var w struct { @@ -235,16 +244,21 @@ func (g *Gateway) WithAPIKey(key string) *Gateway { return g } -// Complete performs one non-streaming POST {baseURL}/chat/completions and returns the parsed -// response. It maps failure modes to typed sentinels: -// - deadline/timeout (context or transport) → ErrTimeout (504) -// - non-2xx status, transport error, bad JSON body → ErrGateway (502) +// Complete performs one POST {baseURL}/chat/completions and returns the whole answer as one ChatResponse. +// The request asks for a stream that ends with the usage (stream, stream_options.include_usage): the dev +// gateway reports a call's whole input only on that path (issue #30 item 8). The stream is read to its end +// and folded back (foldStream), so the agents, the recorder and replay see what a non-streamed answer +// gives; a gateway that ignores stream and answers one JSON body is read as before. It maps failure modes +// to typed sentinels: +// - deadline/timeout (context or transport), mid-stream too → ErrTimeout (504) +// - non-2xx status, transport error, a body that is neither an answer nor a finished stream, +// an error event mid-stream → ErrGateway (502) // -// The request always carries stream:false. The caller's ctx is honored; Complete additionally -// bounds the call by g.timeout so a hung gateway cannot outlive the deadline even if the caller's -// ctx has none. +// The caller's ctx is honored; Complete additionally bounds the call by g.timeout so a hung gateway +// cannot outlive the deadline even if the caller's ctx has none. func (g *Gateway) Complete(ctx context.Context, req ChatRequest) (ChatResponse, error) { - req.Stream = false + req.Stream = true + req.StreamOptions = &StreamOptions{IncludeUsage: true} body, err := json.Marshal(req) if err != nil { @@ -282,22 +296,140 @@ func (g *Gateway) Complete(ctx context.Context, req ChatRequest) (ChatResponse, } if resp.StatusCode < 200 || resp.StatusCode >= 300 { - // Truncate the upstream body so a large error page never bloats the wrapped error/log. - snippet := string(respBody) - // A gateway may echo the rejected credential; scrub it BEFORE truncating (a cut could - // otherwise leave a key prefix behind) so the error/log never carries the key. - if g.apiKey != "" { - snippet = strings.ReplaceAll(snippet, g.apiKey, "[redacted]") - } - if len(snippet) > 512 { - snippet = snippet[:512] - } - return ChatResponse{}, fmt.Errorf("ai.Gateway.Complete: %w: status %d: %s", ErrGateway, resp.StatusCode, snippet) + return ChatResponse{}, fmt.Errorf("ai.Gateway.Complete: %w: status %d: %s", ErrGateway, resp.StatusCode, g.snippet(string(respBody))) } + // A gateway that ignores stream answers one JSON object; an event stream never starts with "{". var out ChatResponse - if err := json.Unmarshal(respBody, &out); err != nil { - return ChatResponse{}, fmt.Errorf("ai.Gateway.Complete: %w: decode response: %v", ErrGateway, err) + if b := bytes.TrimSpace(respBody); len(b) > 0 && b[0] == '{' { + err = json.Unmarshal(b, &out) + } else { + out, err = foldStream(respBody) + } + if err != nil { + return ChatResponse{}, fmt.Errorf("ai.Gateway.Complete: %w: %s", ErrGateway, g.snippet(err.Error())) + } + return out, nil +} + +// snippet is upstream text fit for an error and a log line. A gateway may echo the rejected credential; +// it is scrubbed BEFORE truncating (a cut could otherwise leave a key prefix behind), then cut to 512 bytes +// so a large error page never bloats the wrapped error. +func (g *Gateway) snippet(s string) string { + if g.apiKey != "" { + s = strings.ReplaceAll(s, g.apiKey, "[redacted]") + } + if len(s) > 512 { + s = s[:512] + } + return s +} + +// streamChunk is one data event of a streamed completion (OpenAI's chat.completion.chunk). Each choice +// carries a delta: the next piece of the message, and of each tool call by its index. +type streamChunk struct { + Choices []struct { + Index int `json:"index"` + Delta struct { + Role string `json:"role"` + Content string `json:"content"` + Refusal string `json:"refusal"` + ToolCalls []struct { + Index int `json:"index"` + ID string `json:"id"` + Type string `json:"type"` + Function ToolCallFunc `json:"function"` + } `json:"tool_calls"` + } `json:"delta"` + FinishReason string `json:"finish_reason"` + StopReason any `json:"stop_reason"` + } `json:"choices"` + Usage *Usage `json:"usage"` + StopReason any `json:"stop_reason"` + Error json.RawMessage `json:"error"` +} + +// foldStream folds a streamed completion into the ChatResponse a non-streamed one gives: the text and the +// refusal text joined, each tool call's arguments joined by its index (its id, type and name are taken as +// sent, so a gateway that repeats them on every piece, as 9router does, is read right), the last finish +// reason and stop reasons, and the last usage the stream sent (OpenAI sends it once, after the finish). +// A stream is finished by [DONE] or a finish reason (9router sends no [DONE]); one that stops before +// either, or that sends an error event, is an error: its partial tool calls are not an answer. +// +// ponytail: one "data:" line is one event; a gateway that splits an event over several data lines (the +// SSE spec allows it, no OpenAI-compatible gateway does) is read as broken JSON, a 502. +func foldStream(body []byte) (ChatResponse, error) { + var ( + out ChatResponse + choice *ChatChoice + slot = map[int]int{} // a tool call's stream index → its place in the message + done bool + ) + for _, line := range strings.Split(string(body), "\n") { + data, ok := strings.CutPrefix(strings.TrimRight(line, "\r"), "data:") + if !ok { + continue // a blank line, a comment, an event name or id + } + if data = strings.TrimSpace(data); data == "[DONE]" { + done = true + break + } + var c streamChunk + if err := json.Unmarshal([]byte(data), &c); err != nil { + return ChatResponse{}, fmt.Errorf("decode stream chunk: %w", err) + } + if len(c.Error) > 0 && string(c.Error) != "null" { + return ChatResponse{}, fmt.Errorf("error event mid-stream: %s", c.Error) + } + if c.Usage != nil { + out.Usage = *c.Usage + } + if c.StopReason != nil { + out.StopReason = c.StopReason + } + for _, ch := range c.Choices { + if ch.Index != 0 { + continue // the loop only ever reads choices[0] + } + if choice == nil { + choice = &ChatChoice{Message: ChatMessage{Role: "assistant"}} + } + if ch.Delta.Role != "" { + choice.Message.Role = ch.Delta.Role + } + choice.Message.Content += ch.Delta.Content + choice.Message.Refusal += ch.Delta.Refusal + for _, tc := range ch.Delta.ToolCalls { + i, seen := slot[tc.Index] + if !seen { + i, slot[tc.Index] = len(choice.Message.ToolCalls), len(choice.Message.ToolCalls) + choice.Message.ToolCalls = append(choice.Message.ToolCalls, ToolCall{Type: "function"}) + } + call := &choice.Message.ToolCalls[i] + if tc.ID != "" { + call.ID = tc.ID + } + if tc.Type != "" { + call.Type = tc.Type + } + if tc.Function.Name != "" { + call.Function.Name = tc.Function.Name + } + call.Function.Arguments += tc.Function.Arguments + } + if ch.FinishReason != "" { + choice.FinishReason = ch.FinishReason + } + if ch.StopReason != nil { + choice.StopReason = ch.StopReason + } + } + } + if !done && out.StopReason == nil && (choice == nil || choice.FinishReason == "") { + return ChatResponse{}, errors.New("the stream ended before the answer finished") + } + if choice != nil { + out.Choices = []ChatChoice{*choice} } return out, nil } diff --git a/nexus-platform/internal/ai/gateway_stream_test.go b/nexus-platform/internal/ai/gateway_stream_test.go new file mode 100644 index 0000000..bc8177a --- /dev/null +++ b/nexus-platform/internal/ai/gateway_stream_test.go @@ -0,0 +1,321 @@ +package ai + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "nexus/platform/internal/config" +) + +// Issue #30 item 8: the dev gateway (9router) reports a call's whole input only on its streaming path, so the +// gateway asks for a stream with usage and folds it back into the one ChatResponse the agents read. These tests +// pin that fold: text, tool calls, usage, every refusal shape, an error, a cut and a timeout in mid-stream. + +// sse is a streamed answer as a gateway sends it: one "data:" event per chunk, then the end marker. +func sse(chunks ...string) string { + var b strings.Builder + for _, c := range chunks { + b.WriteString("data: " + c + "\n\n") + } + return b.String() + "data: [DONE]\n\n" +} + +// streamingGateway is a Gateway whose server streams body as an event stream. +func streamingGateway(t *testing.T, timeout time.Duration, body string) *Gateway { + t.Helper() + g, _ := newMockGateway(t, timeout, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte(body)) + }) + return g +} + +// The request asks for a stream that ends with the usage: stream true, stream_options.include_usage true. +func TestGateway_Complete_RequestsAStreamWithUsage(t *testing.T) { + var got map[string]json.RawMessage + g, _ := newMockGateway(t, 5*time.Second, func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&got) + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte(sse(`{"choices":[{"index":0,"delta":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}`))) + }) + if _, err := g.Complete(context.Background(), ChatRequest{Model: "m", Messages: []ChatMessage{{Role: "user", Content: "hi"}}}); err != nil { + t.Fatalf("Complete: %v", err) + } + if string(got["stream"]) != "true" || strings.ReplaceAll(string(got["stream_options"]), " ", "") != `{"include_usage":true}` { + t.Fatalf("stream = %s, stream_options = %s; want true and {\"include_usage\":true}", got["stream"], got["stream_options"]) + } +} + +func TestGateway_Complete_FoldsTheStream(t *testing.T) { + for _, tc := range []struct { + name, body string + want ChatResponse + }{ + {"text in pieces, then a usage-only chunk (OpenAI)", + sse(`{"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}],"usage":null}`, + `{"choices":[{"index":0,"delta":{"content":"Tôi đề xuất "},"finish_reason":null}],"usage":null}`, + `{"choices":[{"index":0,"delta":{"content":"entity patients."},"finish_reason":null}],"usage":null}`, + `{"choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":null}`, + `{"choices":[],"usage":{"prompt_tokens":20000,"completion_tokens":300,"total_tokens":20300,"prompt_tokens_details":{"cached_tokens":18000,"cache_creation_tokens":1500}}}`), + ChatResponse{Choices: []ChatChoice{{Message: ChatMessage{Role: "assistant", Content: "Tôi đề xuất entity patients."}, FinishReason: "stop"}}, + Usage: Usage{PromptTokens: 20000, CompletionTokens: 300, TotalTokens: 20300, CacheReadTokens: 18000, CacheWriteTokens: 1500}}}, + {"two tool calls, arguments in pieces, the id repeated on every piece (9router)", + sse(`{"choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}`, + `{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"toolu_1","type":"function","function":{"name":"propose_entity","arguments":""}}]},"finish_reason":null}]}`, + `{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"toolu_1","function":{"arguments":"{\"name\":"}}]},"finish_reason":null}]}`, + `{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"toolu_1","function":{"arguments":"\"tickets\"}"}}]},"finish_reason":null}]}`, + `{"choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"id":"toolu_2","type":"function","function":{"name":"list_entities","arguments":"{}"}}]},"finish_reason":null}]}`, + `{"choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":36836,"completion_tokens":120,"total_tokens":36956}}`), + ChatResponse{Choices: []ChatChoice{{Message: ChatMessage{Role: "assistant", ToolCalls: []ToolCall{ + {ID: "toolu_1", Type: "function", Function: ToolCallFunc{Name: "propose_entity", Arguments: `{"name":"tickets"}`}}, + {ID: "toolu_2", Type: "function", Function: ToolCallFunc{Name: "list_entities", Arguments: "{}"}}, + }}, FinishReason: "tool_calls"}}, + Usage: Usage{PromptTokens: 36836, CompletionTokens: 120, TotalTokens: 36956}}}, + {"no [DONE], usage on the finish chunk: 9router's stream as recorded on 2026-10-27", + "data: " + `{"id":"chatcmpl-msg_011","object":"chat.completion.chunk","created":1793086291,"model":"claude-opus-5-5","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}` + "\n\n" + + "data: " + `{"id":"chatcmpl-msg_011","object":"chat.completion.chunk","created":1793086291,"model":"claude-opus-5-5","choices":[{"index":0,"delta":{"content":"ok"},"finish_reason":null}]}` + "\n\n" + + "data: " + `{"id":"chatcmpl-msg_011","object":"chat.completion.chunk","created":1793086291,"model":"claude-opus-5-5","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":18836,"completion_tokens":4,"total_tokens":18840}}` + "\n\n", + ChatResponse{Choices: []ChatChoice{{Message: ChatMessage{Role: "assistant", Content: "ok"}, FinishReason: "stop"}}, + Usage: Usage{PromptTokens: 18836, CompletionTokens: 4, TotalTokens: 18840}}}, + {"comments, event names and ids between the chunks are not data", + ": keep-alive\n\nevent: message\nid: 1\ndata: " + `{"choices":[{"index":0,"delta":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}` + "\n\n: bye\n\ndata: [DONE]\n\n", + ChatResponse{Choices: []ChatChoice{{Message: ChatMessage{Role: "assistant", Content: "ok"}, FinishReason: "stop"}}}}, + {"data: without a space, CRLF line ends", + "data:" + `{"choices":[{"index":0,"delta":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],"usage":{"input_tokens":2,"output_tokens":1,"cache_read_input_tokens":30000}}` + "\r\n\r\ndata:[DONE]\r\n\r\n", + ChatResponse{Choices: []ChatChoice{{Message: ChatMessage{Role: "assistant", Content: "ok"}, FinishReason: "stop"}}, + Usage: Usage{PromptTokens: 30002, CompletionTokens: 1, TotalTokens: 30003, CacheReadTokens: 30000}}}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := streamingGateway(t, 5*time.Second, tc.body).Complete(context.Background(), ChatRequest{Model: "m"}) + if err != nil { + t.Fatalf("Complete: %v", err) + } + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("response =\n%+v\nwant\n%+v", got, tc.want) + } + }) + } +} + +// A streamed answer reads exactly as the same answer sent whole: the agents see one ChatResponse, so a +// refusal, a finish reason or a tool call means the same thing on both paths (#69's Refused() mappings). +func TestGateway_Complete_StreamReadsAsTheWholeAnswer(t *testing.T) { + const usage = `"usage":{"prompt_tokens":2046,"completion_tokens":0}` + for _, tc := range []struct { + name, whole string + stream []string + refused bool + }{ + {"finish_reason refusal, no text", + `{"choices":[{"message":{"role":"assistant","content":null},"finish_reason":"refusal"}],` + usage + `}`, + []string{`{"choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}`, `{"choices":[{"index":0,"delta":{},"finish_reason":"refusal"}],` + usage + `}`}, true}, + {"stop_reason refusal on the choice", + `{"choices":[{"message":{"role":"assistant","content":""},"finish_reason":"stop","stop_reason":"refusal"}],` + usage + `}`, + []string{`{"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}`, `{"choices":[{"index":0,"delta":{},"finish_reason":"stop","stop_reason":"refusal"}],` + usage + `}`}, true}, + {"finish_reason content_filter", + `{"choices":[{"message":{"role":"assistant","content":null},"finish_reason":"content_filter"}],` + usage + `}`, + []string{`{"choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":"content_filter"}],` + usage + `}`}, true}, + {"OpenAI refusal text, streamed in pieces", + `{"choices":[{"message":{"role":"assistant","content":null,"refusal":"I can't help with that."},"finish_reason":"stop"}],` + usage + `}`, + []string{`{"choices":[{"index":0,"delta":{"role":"assistant","refusal":"I can't "},"finish_reason":null}]}`, + `{"choices":[{"index":0,"delta":{"refusal":"help with that."},"finish_reason":null}]}`, + `{"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}`, `{"choices":[],` + usage + `}`}, true}, + {"top-level stop_reason refusal", + `{"choices":[{"message":{"role":"assistant","content":""},"finish_reason":"stop"}],"stop_reason":"refusal",` + usage + `}`, + []string{`{"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop"}],"stop_reason":"refusal",` + usage + `}`}, true}, + {"top-level stop_reason refusal, no choices", + `{"choices":[],"stop_reason":"refusal",` + usage + `}`, + []string{`{"choices":[],"stop_reason":"refusal",` + usage + `}`}, true}, + {"stop_reason is a token id (vLLM)", + `{"choices":[{"message":{"role":"assistant","content":"ok"},"finish_reason":"stop","stop_reason":128001}],` + usage + `}`, + []string{`{"choices":[{"index":0,"delta":{"role":"assistant","content":"ok"},"finish_reason":null}]}`, `{"choices":[{"index":0,"delta":{},"finish_reason":"stop","stop_reason":128001}],` + usage + `}`}, false}, + {"a tool call", + `{"choices":[{"message":{"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"list_entities","arguments":"{}"}}]},"finish_reason":"tool_calls"}],` + usage + `}`, + []string{`{"choices":[{"index":0,"delta":{"role":"assistant","content":"","tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"list_entities","arguments":"{"}}]},"finish_reason":null}]}`, + `{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"}"}}]},"finish_reason":"tool_calls"}],` + usage + `}`}, false}, + {"max_tokens cut", `{"choices":[{"message":{"role":"assistant","content":"Bảng"},"finish_reason":"length"}],` + usage + `}`, + []string{`{"choices":[{"index":0,"delta":{"role":"assistant","content":"Bảng"},"finish_reason":"length"}],` + usage + `}`}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + whole, err := streamingGatewayJSON(t, tc.whole).Complete(context.Background(), ChatRequest{Model: "m"}) + if err != nil { + t.Fatalf("whole: %v", err) + } + streamed, err := streamingGateway(t, 5*time.Second, sse(tc.stream...)).Complete(context.Background(), ChatRequest{Model: "m"}) + if err != nil { + t.Fatalf("streamed: %v", err) + } + if len(whole.Choices) == 0 { + whole.Choices = nil // "choices":[] and a stream with no choice say the same + } + if !reflect.DeepEqual(streamed, whole) { + t.Fatalf("streamed =\n%+v\nwhole =\n%+v", streamed, whole) + } + if streamed.Refused() != tc.refused { + t.Fatalf("Refused() = %v; want %v", streamed.Refused(), tc.refused) + } + }) + } +} + +// streamingGatewayJSON is a Gateway whose server ignores stream and answers one JSON body, as a gateway +// without streaming does. +func streamingGatewayJSON(t *testing.T, body string) *Gateway { + t.Helper() + g, _ := newMockGateway(t, 5*time.Second, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + }) + return g +} + +// A stream that fails part-way is a failed call, never a half answer: an error event, a stream cut before +// the answer finished, a chunk that is not JSON and a cut connection are gateway errors (502); a gateway that +// stops sending is a timeout (504). The key never appears in the error. +func TestGateway_Complete_StreamFailures(t *testing.T) { + const key = "sk-stream-secret-77" + partial := `{"choices":[{"index":0,"delta":{"role":"assistant","tool_calls":[{"index":0,"id":"c1","type":"function","function":{"name":"propose_entity","arguments":"{\"na"}}]},"finish_reason":null}]}` + for _, tc := range []struct { + name string + handler http.HandlerFunc + want error + says string + }{ + {"an error event mid-stream", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte("data: " + partial + "\n\ndata: {\"error\":{\"message\":\"upstream overloaded for " + key + "\",\"type\":\"overloaded_error\"}}\n\n")) + }, ErrGateway, "upstream overloaded"}, + {"the stream ends before the answer finished", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte("data: " + partial + "\n\n")) + }, ErrGateway, "ended before"}, + {"a chunk that is not JSON", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte("data: " + partial + "\n\ndata: {not json\n\n")) + }, ErrGateway, "decode"}, + {"the connection is cut mid-stream", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Content-Length", "100000") // promise more than is sent: the client reads an unexpected EOF + _, _ = w.Write([]byte("data: " + partial + "\n\n")) + }, ErrGateway, "read"}, + {"the gateway stops sending mid-stream", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte("data: " + partial + "\n\n")) + w.(http.Flusher).Flush() + select { + case <-r.Context().Done(): + case <-time.After(2 * time.Second): + } + }, ErrTimeout, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + g, _ := newMockGateway(t, 200*time.Millisecond, tc.handler) + start := time.Now() + _, err := g.WithAPIKey(key).Complete(context.Background(), ChatRequest{Model: "m"}) + if !errors.Is(err, tc.want) { + t.Fatalf("Complete() error = %v; want %v", err, tc.want) + } + if tc.want == ErrGateway && errors.Is(err, ErrTimeout) { + t.Fatalf("error = %v; a failed stream is not a timeout", err) + } + if !strings.Contains(err.Error(), tc.says) { + t.Fatalf("error = %q; want it to say %q", err, tc.says) + } + if strings.Contains(err.Error(), key) { + t.Fatalf("error leaks the API key: %q", err) + } + if time.Since(start) > time.Second { + t.Fatalf("Complete() took %v; want it to fail at the 200ms deadline at the latest", time.Since(start)) + } + }) + } +} + +// Recording a streamed call writes the folded answer, the shape every cassette has always had, so replay +// reads it like any older line: the same choices back, usage zeroed. +func TestRecord_StreamedCallReplays(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte(sse( + `{"choices":[{"index":0,"delta":{"role":"assistant","tool_calls":[{"index":0,"id":"toolu_1","type":"function","function":{"name":"list_entities","arguments":""}}]},"finish_reason":null}]}`, + `{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"toolu_1","function":{"arguments":"{}"}}]},"finish_reason":"tool_calls"}]}`, + `{"choices":[],"usage":{"prompt_tokens":36836,"completion_tokens":120,"prompt_tokens_details":{"cached_tokens":34960,"cache_creation_tokens":472}}}`))) + })) + t.Cleanup(srv.Close) + path := filepath.Join(t.TempDir(), "streamed.jsonl") + rec, err := NewCompleter(config.AIConfig{GatewayMode: config.AIGatewayRecord, Cassette: path, BaseURL: srv.URL, TimeoutSec: 5}) + if err != nil { + t.Fatalf("NewCompleter(record): %v", err) + } + live, err := rec.Complete(context.Background(), ask("q1")) + if err != nil { + t.Fatalf("record: %v", err) + } + if want := (Usage{PromptTokens: 36836, CompletionTokens: 120, TotalTokens: 36956, CacheReadTokens: 34960, CacheWriteTokens: 472}); live.Usage != want { + t.Fatalf("recorded usage = %+v; want %+v", live.Usage, want) + } + lines := readCassette(t, path) + if len(lines) != 1 || !reflect.DeepEqual(lines[0].Response, live) || lines[0].Request.Stream { + t.Fatalf("cassette = %+v; want one line holding the folded answer and the request as the agent sent it", lines) + } + + rep, err := NewCompleter(config.AIConfig{GatewayMode: config.AIGatewayReplay, Cassette: path}) + if err != nil { + t.Fatalf("NewCompleter(replay): %v", err) + } + got, err := rep.Complete(context.Background(), ask("q1")) + if err != nil { + t.Fatalf("replay: %v", err) + } + want := live + want.Usage = Usage{} + if !reflect.DeepEqual(got, want) { + t.Fatalf("replayed = %+v; want %+v", got, want) + } +} + +// The cassettes recorded before the stream (non-streaming answers, kept in git and replayed by CI's demo-loop +// jobs) load and answer every recorded call with its recorded choices, unchanged. +func TestReplay_TheNonStreamingCassettesStillAnswer(t *testing.T) { + for _, name := range []string{"demo-loop.jsonl", "demo-loop-en.jsonl"} { + t.Run(name, func(t *testing.T) { + path := filepath.Join("..", "..", "..", "cassettes", name) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + var entries []cassetteEntry + for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") { + var e cassetteEntry + if err := json.Unmarshal([]byte(line), &e); err != nil { + t.Fatalf("line: %v", err) + } + entries = append(entries, e) + } + rep, err := NewCompleter(config.AIConfig{GatewayMode: config.AIGatewayReplay, Cassette: path}) + if err != nil { + t.Fatalf("NewCompleter(replay): %v", err) + } + for i, e := range entries { + got, err := rep.Complete(context.Background(), e.Request) + if err != nil { + t.Fatalf("call %d: %v", i+1, err) + } + want := e.Response + want.Usage = Usage{} + if !reflect.DeepEqual(got, want) { + t.Fatalf("call %d = %+v; want the recorded answer %+v", i+1, got, want) + } + } + }) + } +} diff --git a/nexus-platform/internal/ai/gateway_test.go b/nexus-platform/internal/ai/gateway_test.go index 1e966c7..008b37a 100644 --- a/nexus-platform/internal/ai/gateway_test.go +++ b/nexus-platform/internal/ai/gateway_test.go @@ -25,11 +25,11 @@ func TestGateway_Complete_ToolCallsResponse(t *testing.T) { if r.URL.Path != "/v1/chat/completions" { t.Errorf("path = %q; want /v1/chat/completions", r.URL.Path) } - // Assert the request never streams. + // The request asks for a stream; this gateway ignores it and answers one JSON body, which is read as before. var req ChatRequest _ = json.NewDecoder(r.Body).Decode(&req) - if req.Stream { - t.Error("request Stream = true; want false") + if !req.Stream { + t.Error("request Stream = false; want true") } w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{ diff --git a/nexus-platform/internal/handlers/ai_refusal_test.go b/nexus-platform/internal/handlers/ai_refusal_test.go index 1b6b78d..240d46b 100644 --- a/nexus-platform/internal/handlers/ai_refusal_test.go +++ b/nexus-platform/internal/handlers/ai_refusal_test.go @@ -31,12 +31,23 @@ func (groundingOnlyRecordReader) ViewableEntities(context.Context, services.Reco } // An API-level refusal (HTTP 200, finish_reason "refusal", no text) reaches the panel as outcome "refused" on -// both chat routes, through the real gateway client and the real agents (issue #30 item 5). The turn is stored +// both chat routes, through the real gateway client and the real agents (issue #30 item 5), whether the +// gateway answers in one body or streams (issue #30 item 8: the client asks for a stream). The turn is stored // like any answer, so the thread goes on. func TestChatRoutes_ModelRefusal_200Refused(t *testing.T) { + for wire, body := range map[string]string{ + "whole": `{"choices":[{"message":{"role":"assistant","content":null},"finish_reason":"refusal"}], + "usage":{"prompt_tokens":2046,"completion_tokens":0,"total_tokens":2046}}`, + "streamed": "data: " + `{"choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}` + "\n\n" + + "data: " + `{"choices":[{"index":0,"delta":{},"finish_reason":"refusal"}],"usage":{"prompt_tokens":2046,"completion_tokens":0,"total_tokens":2046}}` + "\n\n", + } { + t.Run(wire, func(t *testing.T) { refusedOnBothRoutes(t, body) }) + } +} + +func refusedOnBothRoutes(t *testing.T, body string) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":null},"finish_reason":"refusal"}], - "usage":{"prompt_tokens":2046,"completion_tokens":0,"total_tokens":2046}}`)) + _, _ = w.Write([]byte(body)) })) t.Cleanup(srv.Close) gw := ai.NewGateway(srv.URL, 5*time.Second) From fca5d0dd0619f8c4ba3ba8272c27f3b482d21b1b Mon Sep 17 00:00:00 2001 From: doxuta Date: Tue, 27 Oct 2026 14:42:11 +0700 Subject: [PATCH 2/7] fix(ai): a streamed tool call without arguments reads as {} (#30 item 8) Claude streams no argument piece for a call that takes none, so the fold gave "" and 9router refused the next call that sent it back (400 tool_use.input: Input should be an object). A whole answer carries "{}"; the fold now does too. Found by the first live recorded turn. Co-Authored-By: Claude Opus 5.5 --- nexus-platform/internal/ai/gateway.go | 11 +++++++++-- nexus-platform/internal/ai/gateway_stream_test.go | 11 +++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/nexus-platform/internal/ai/gateway.go b/nexus-platform/internal/ai/gateway.go index 588a9f6..e63e206 100644 --- a/nexus-platform/internal/ai/gateway.go +++ b/nexus-platform/internal/ai/gateway.go @@ -350,8 +350,8 @@ type streamChunk struct { } // foldStream folds a streamed completion into the ChatResponse a non-streamed one gives: the text and the -// refusal text joined, each tool call's arguments joined by its index (its id, type and name are taken as -// sent, so a gateway that repeats them on every piece, as 9router does, is read right), the last finish +// refusal text joined, each tool call's arguments joined by its index ("{}" when none came; its id, type and +// name are taken as sent, so a gateway that repeats them on every piece, as 9router does, is read right), the last finish // reason and stop reasons, and the last usage the stream sent (OpenAI sends it once, after the finish). // A stream is finished by [DONE] or a finish reason (9router sends no [DONE]); one that stops before // either, or that sends an error event, is an error: its partial tool calls are not an answer. @@ -429,6 +429,13 @@ func foldStream(body []byte) (ChatResponse, error) { return ChatResponse{}, errors.New("the stream ended before the answer finished") } if choice != nil { + // A call without arguments streams no piece of them; a whole answer sends "{}", and the gateway + // refuses "" when the call is sent back (9router: "tool_use.input: Input should be an object"). + for i := range choice.Message.ToolCalls { + if strings.TrimSpace(choice.Message.ToolCalls[i].Function.Arguments) == "" { + choice.Message.ToolCalls[i].Function.Arguments = "{}" + } + } out.Choices = []ChatChoice{*choice} } return out, nil diff --git a/nexus-platform/internal/ai/gateway_stream_test.go b/nexus-platform/internal/ai/gateway_stream_test.go index bc8177a..7e168ba 100644 --- a/nexus-platform/internal/ai/gateway_stream_test.go +++ b/nexus-platform/internal/ai/gateway_stream_test.go @@ -80,6 +80,17 @@ func TestGateway_Complete_FoldsTheStream(t *testing.T) { {ID: "toolu_2", Type: "function", Function: ToolCallFunc{Name: "list_entities", Arguments: "{}"}}, }}, FinishReason: "tool_calls"}}, Usage: Usage{PromptTokens: 36836, CompletionTokens: 120, TotalTokens: 36956}}}, + {"a tool call without arguments streams no piece of them: read as {}, as a whole answer sends it", + // 9router, 2026-10-27: list_packs came back with arguments "" and the next call, which sent it back, + // failed with 400 "tool_use.input: Input should be an object". + sse(`{"choices":[{"index":0,"delta":{"role":"assistant","tool_calls":[{"index":0,"id":"toolu_1","type":"function","function":{"name":"list_roles","arguments":""}}]},"finish_reason":null}]}`, + `{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"toolu_1","function":{"arguments":"{\"grants\": true}"}}]},"finish_reason":null}]}`, + `{"choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"id":"toolu_2","type":"function","function":{"name":"list_packs","arguments":""}}]},"finish_reason":null}]}`, + `{"choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]}`), + ChatResponse{Choices: []ChatChoice{{Message: ChatMessage{Role: "assistant", ToolCalls: []ToolCall{ + {ID: "toolu_1", Type: "function", Function: ToolCallFunc{Name: "list_roles", Arguments: `{"grants": true}`}}, + {ID: "toolu_2", Type: "function", Function: ToolCallFunc{Name: "list_packs", Arguments: "{}"}}, + }}, FinishReason: "tool_calls"}}}}, {"no [DONE], usage on the finish chunk: 9router's stream as recorded on 2026-10-27", "data: " + `{"id":"chatcmpl-msg_011","object":"chat.completion.chunk","created":1793086291,"model":"claude-opus-5-5","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}` + "\n\n" + "data: " + `{"id":"chatcmpl-msg_011","object":"chat.completion.chunk","created":1793086291,"model":"claude-opus-5-5","choices":[{"index":0,"delta":{"content":"ok"},"finish_reason":null}]}` + "\n\n" + From e4484017b8e2874c0de9f060d0113b5f2d85dc88 Mon Sep 17 00:00:00 2001 From: doxuta Date: Tue, 27 Oct 2026 14:49:52 +0700 Subject: [PATCH 3/7] refactor(ai): foldStream errors carry their package prefix; doc reflowed Co-Authored-By: Claude Opus 5.5 --- nexus-platform/internal/ai/gateway.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/nexus-platform/internal/ai/gateway.go b/nexus-platform/internal/ai/gateway.go index e63e206..8594868 100644 --- a/nexus-platform/internal/ai/gateway.go +++ b/nexus-platform/internal/ai/gateway.go @@ -350,11 +350,11 @@ type streamChunk struct { } // foldStream folds a streamed completion into the ChatResponse a non-streamed one gives: the text and the -// refusal text joined, each tool call's arguments joined by its index ("{}" when none came; its id, type and -// name are taken as sent, so a gateway that repeats them on every piece, as 9router does, is read right), the last finish -// reason and stop reasons, and the last usage the stream sent (OpenAI sends it once, after the finish). -// A stream is finished by [DONE] or a finish reason (9router sends no [DONE]); one that stops before -// either, or that sends an error event, is an error: its partial tool calls are not an answer. +// refusal text joined; each tool call's arguments joined by its index, "{}" when none came, its id, type and +// name taken as sent (9router repeats the id on every piece); the last finish reason and stop reasons; and +// the last usage the stream sent (OpenAI sends it once, after the finish). [DONE], a finish reason or a +// top-level stop reason ends the answer (9router sends no [DONE]); a stream that stops before one, or that +// sends an error event, is an error: its partial tool calls are not an answer. // // ponytail: one "data:" line is one event; a gateway that splits an event over several data lines (the // SSE spec allows it, no OpenAI-compatible gateway does) is read as broken JSON, a 502. @@ -376,10 +376,10 @@ func foldStream(body []byte) (ChatResponse, error) { } var c streamChunk if err := json.Unmarshal([]byte(data), &c); err != nil { - return ChatResponse{}, fmt.Errorf("decode stream chunk: %w", err) + return ChatResponse{}, fmt.Errorf("ai.foldStream: decode stream chunk: %w", err) } if len(c.Error) > 0 && string(c.Error) != "null" { - return ChatResponse{}, fmt.Errorf("error event mid-stream: %s", c.Error) + return ChatResponse{}, fmt.Errorf("ai.foldStream: error event mid-stream: %s", c.Error) } if c.Usage != nil { out.Usage = *c.Usage @@ -402,7 +402,8 @@ func foldStream(body []byte) (ChatResponse, error) { for _, tc := range ch.Delta.ToolCalls { i, seen := slot[tc.Index] if !seen { - i, slot[tc.Index] = len(choice.Message.ToolCalls), len(choice.Message.ToolCalls) + i = len(choice.Message.ToolCalls) + slot[tc.Index] = i choice.Message.ToolCalls = append(choice.Message.ToolCalls, ToolCall{Type: "function"}) } call := &choice.Message.ToolCalls[i] @@ -426,7 +427,7 @@ func foldStream(body []byte) (ChatResponse, error) { } } if !done && out.StopReason == nil && (choice == nil || choice.FinishReason == "") { - return ChatResponse{}, errors.New("the stream ended before the answer finished") + return ChatResponse{}, errors.New("ai.foldStream: the stream ended before the answer finished") } if choice != nil { // A call without arguments streams no piece of them; a whole answer sends "{}", and the gateway From 0732d8027c8f91c14b591028c36e9ae0b4766571 Mon Sep 17 00:00:00 2001 From: doxuta Date: Tue, 27 Oct 2026 14:49:52 +0700 Subject: [PATCH 4/7] docs(bench): the measured cost of one app with streamed usage (#30 item 8) One recorded app-helpdesk turn on Claude Opus 5.5 through 9router: 107,305 input tokens (179 uncached, 103,158 cache read, 3,968 cache write) and 1,779 output, about $0.08 warm and $0.34 cold at list prices. The streamed prompt total is the whole input plus 9router's 2,000 per call; the cache split comes from 9router's own log because its OpenAI-shape usage drops it. The env example says calls are streamed. Co-Authored-By: Claude Opus 5.5 --- evals/builder-bench/report.md | 38 ++++++++++++++++++++++++++++++++--- nexus-platform/.env.example | 2 ++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/evals/builder-bench/report.md b/evals/builder-bench/report.md index 230e9a7..0e911b5 100644 --- a/evals/builder-bench/report.md +++ b/evals/builder-bench/report.md @@ -75,17 +75,49 @@ turns, including a turn the panel sent on its own. - **The prompt side is a floor.** A model call reported 2,046 prompt tokens on average over the cheap model's 213 calls, and 2,049 over Opus's 65, whatever its size. A one-line test request through the same gateway reported 2,042. A builder call sends about 70 KB (the persona, 26 tool schemas, the workspace's tables). This gateway reports an - uncached slice of the input; the cached bulk is not in the count. + uncached slice of the input; the cached bulk is not in the count. The engine now streams its calls and reads the + whole input (#30 item 8); one app measured that way is below. - **The output side is exact:** 2,666 output tokens per successful app on the cheap model, 2,039 on Opus. - **"KB sent" is the input actually carried:** 14,835 KB for the cheap model's 60 attempts, 4,574 KB for Opus's 20. -- **No dollar figure.** List prices exist, but the input count they would multiply is not known here (cache reads and - writes are not reported), so a dollar amount would be invented. +- **No dollar figure for these runs.** List prices exist, but the input count they would multiply is not known here + (cache reads and writes are not reported), so a dollar amount would be invented. One app priced from a measured + run is below. **Totals.** The 80 attempts recorded 734,940 tokens; stage 1 alone used 93,221, far under the 1.5M that gates stage 2. Before the runs, one probe of the cheap model's id through the gateway used 2,046. One attempt, aborted by a driver bug (fixed in `1853ae6`) and discarded, used 5,626 measured tokens; a third call was cut mid-flight and its usage is unknown. All in, about 742,600 tokens, under the 5M cap. +## What one app really costs (measured 2026-10-27) + +One builder turn on the `app-helpdesk` prompt, after the engine started streaming its model calls with +`stream_options.include_usage` (#30 item 8). Claude Opus 5.5 (`cc/claude-opus-5-5`) through the same local gateway +(9router 0.5.99), on the one-command demo stack in `AI_GATEWAY_MODE=record`, sent as the builder through +`POST /api/v1/admin/ai/builder/chat`. The turn ended `ok` with a draft of every expected kind (entity, app, access) in +three model calls; it was not applied or published, so it is one app's turn, not a bench attempt. + +| call | prompt tokens reported | whole input | uncached | cache read | cache write | output | +|---|---|---|---|---|---|---| +| 1 (list_roles, list_packs) | 35,531 | 33,531 | 175 | 33,356 | 0 | 285 | +| 2 (propose_entity, propose_app, propose_access) | 38,448 | 36,448 | 2 | 33,356 | 3,090 | 772 | +| 3 (the reply) | 39,326 | 37,326 | 2 | 36,446 | 878 | 722 | +| **the turn** | **113,305** (`sys_ai_runs`) | **107,305** | **179** | **103,158** | **3,968** | **1,779** | + +- **What the gateway reports, streamed:** the whole input plus 2,000 per call. 9router adds 2,000 to `prompt_tokens` + on both paths and keeps only `prompt_tokens`, `completion_tokens` and `total_tokens` in its OpenAI-shape usage, so + the cache split never reaches the engine and `sys_ai_runs.cache_read_tokens` / `cache_write_tokens` stay NULL + through it. The split above is 9router's own usage log for the same three calls. A gateway that forwards the + split (OpenAI's `prompt_tokens_details`, LiteLLM's Anthropic fields) fills those columns; the engine reads both. +- **Not streamed** (every attempt in this report), the same three calls would have read 6,179 prompt tokens: the + uncached part plus 2,000 each. The whole input is 17 times that. +- **In dollars,** at Claude Opus 5.5's list prices ($4 / MTok input, $20 output, $0.20 cache read, $5 cache write for + 5 minutes, $8 for 1 hour): 179 uncached $0.0007, 103,158 read $0.0206, 3,968 written $0.0198, 1,779 output + $0.0356, so **about $0.08 for the app** while the cache is warm. 9router marks the conversation tail with the + 5-minute cache, and these writes are the tail; at the 1-hour rate they would make it $0.089. +- **Cold,** the first turn after the cache expires writes the persona and the tool schemas (33,356 tokens, marked + with the 1-hour cache) instead of reading them: $0.26 more, **about $0.34 for the app**. The turn before this one + wrote exactly that. + ## The run - **Product under test:** origin/main `d300dfc` (#21), this branch's base. The bench's commits change no server or web diff --git a/nexus-platform/.env.example b/nexus-platform/.env.example index eaca80e..f6b2605 100644 --- a/nexus-platform/.env.example +++ b/nexus-platform/.env.example @@ -36,6 +36,8 @@ REDIS_URL=redis://127.0.0.1:6379 # To turn it ON, set BOTH of these — a gateway must actually be reachable: #AI_ENABLED=true # Any OpenAI-compatible /v1/chat/completions gateway works (LiteLLM, vLLM, Ollama, a hosted endpoint). +# Calls are streamed with stream_options.include_usage, so the ledger gets the whole input; a gateway +# that ignores stream and answers one JSON body works too. #AI_GATEWAY_BASE_URL=http://127.0.0.1:4000/v1 # Put the real key in .env only (never commit it). The model id must be one the gateway lists (GET /v1/models): #AI_GATEWAY_API_KEY= From 39aaa2a6d868a9de7ac5fff2b715a747aa4ff576 Mon Sep 17 00:00:00 2001 From: doxuta Date: Tue, 27 Oct 2026 15:07:51 +0700 Subject: [PATCH 5/7] fix(ai): size the default daily token budget in whole-input tokens (#30 item 8) Streamed usage moves sys_ai_runs.prompt_tokens from the uncached remainder to the whole input, about 17x on a builder turn (measured 115,084 tokens a turn, was ~8,000). The 200,000 default then admitted 2 builder turns a tenant a day instead of 26. 3,000,000 admits 27, pinned by a test against the measured turn. The budget is documented in .env.example and passed through docker compose. Co-Authored-By: Claude Opus 5.5 --- docker-compose.yml | 2 ++ nexus-platform/.env.example | 5 +++ nexus-platform/internal/ai/budget.go | 9 ++--- nexus-platform/internal/ai/budget_test.go | 40 +++++++++++++++++++++ nexus-platform/internal/config/ai_config.go | 9 +++-- 5 files changed, 59 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e7e2ce8..ca4ad79 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -62,6 +62,8 @@ services: AI_GATEWAY_BASE_URL: ${AI_GATEWAY_BASE_URL:-} AI_GATEWAY_API_KEY: ${AI_GATEWAY_API_KEY:-} AI_MODEL: ${AI_MODEL:-claude-opus-5-5} + # Tokens per tenant per UTC day; empty = the default 3,000,000, ~27 builder turns (nexus-platform/.env.example). + AI_DAILY_TOKEN_BUDGET: ${AI_DAILY_TOKEN_BUDGET:-} # live | record | replay (nexus-platform/.env.example); a cassette path only with record or replay. AI_GATEWAY_MODE: ${AI_GATEWAY_MODE:-live} AI_GATEWAY_CASSETTE: ${AI_GATEWAY_CASSETTE:-} diff --git a/nexus-platform/.env.example b/nexus-platform/.env.example index f6b2605..8ee0173 100644 --- a/nexus-platform/.env.example +++ b/nexus-platform/.env.example @@ -44,6 +44,11 @@ REDIS_URL=redis://127.0.0.1:6379 #AI_MODEL=claude-opus-5-5 # Tool-loop depth per turn (Discovery mode needs ~14; default 12): #AI_MAX_STEPS=14 +# Tokens per tenant per UTC day, every AI call (prompt + output); past it a turn answers 429 +# AI_BUDGET_EXCEEDED until midnight UTC. A call counts its whole input, cached or not: one builder turn +# that makes an app is ~115,000 tokens, so the default 3,000,000 is ~27 a day. 0 turns the cap off. +# One user's ⌘K assistant gets at most ASSISTANT_DAILY_TOKEN_SHARE_PCT (default 50) percent of it. +#AI_DAILY_TOKEN_BUDGET=3000000 # Probe: GET /api/v1/ai/health → {"data":{"enabled":true|false}} — the FE reads this # to show an honest on/off state BEFORE the user types. # Record/replay, for demos, CI and evals without a live model: live (default) | record | replay. diff --git a/nexus-platform/internal/ai/budget.go b/nexus-platform/internal/ai/budget.go index 718b090..4a78a96 100644 --- a/nexus-platform/internal/ai/budget.go +++ b/nexus-platform/internal/ai/budget.go @@ -27,10 +27,11 @@ // Accepted overshoot (spec §12): the check runs PRE-turn against a ledger that only reflects // COMPLETED turns, so every turn admitted while others are still in flight can independently pass // the same budget snapshot before any of them commits usage back. Up to MaxConcurrent turns can be -// in that window at once, each spending up to MaxSteps × MaxTokens — so the day total may overshoot -// the configured budget by at most MaxConcurrent × MaxSteps × MaxTokens (default -// 2 × 12 × 4096 ≈ 98k, against a 200k default budget — MaxSteps was raised 6→12 for Discovery -// M10, so one worst-case turn is ~49k). That is bounded and not worth a mid-turn abort. +// in that window at once, each spending up to MaxSteps calls of whole input plus MaxTokens output — so +// the day total may overshoot the configured budget by MaxConcurrent such turns. A builder call's input +// is ~35-40k tokens (evals/builder-bench/report.md), so one worst-case 12-step turn (MaxSteps was raised +// 6→12 for Discovery M10) is ~0.5M and the overshoot ~1M, against a 3M default budget. That is bounded +// and not worth a mid-turn abort. package ai import ( diff --git a/nexus-platform/internal/ai/budget_test.go b/nexus-platform/internal/ai/budget_test.go index 8b71752..eacefb0 100644 --- a/nexus-platform/internal/ai/budget_test.go +++ b/nexus-platform/internal/ai/budget_test.go @@ -216,6 +216,46 @@ func TestBudgetGuard_DailyBudget(t *testing.T) { } } +// measuredBuilderTurn is the sys_ai_runs row of one real builder turn that made an app, its calls streamed +// through the dev gateway (evals/builder-bench/report.md, "What one app really costs"): 113,305 prompt tokens +// (the whole input of three calls, cache reads and writes included) + 1,779 output. +const measuredBuilderTurn = 113305 + 1779 + +// Issue #30 item 8 moved prompt_tokens from the uncached remainder to the whole input, about 17 times more on a +// builder turn, so the default daily budget is sized in that unit. It admits the working day it admitted before +// the stream: 26 turns at the ~8,000 tokens a turn recorded then under 200,000; 27 at the measured 115,084 now. +// A third turn a day (what 200,000 admits now) would 429 AI_BUDGET_EXCEEDED a live demo until UTC midnight. +func TestBudgetGuard_DefaultBudgetAdmitsADayOfBuilderTurns(t *testing.T) { + for _, k := range []string{"AI_DAILY_TOKEN_BUDGET", "AI_GATEWAY_MODE", "AI_GATEWAY_CASSETTE"} { + t.Setenv(k, "") + } + loaded, err := config.LoadAIConfig() + if err != nil { + t.Fatalf("LoadAIConfig: %v", err) + } + cfg := guardCfg() // the rate and concurrency gates out of the way: the budget alone decides + cfg.DailyTokenBudget = loaded.DailyTokenBudget + ledger := &fakeTokenLedger{} + g := newBudgetGuard(cfg, ledger, nil, time.Now) + + turns := 0 + for ; turns < 1000; turns++ { + release, err := g.Acquire(context.Background(), "tenant-a") + if errors.Is(err, ErrBudgetExceeded) { + break + } + if err != nil { + t.Fatalf("turn %d: Acquire error = %v", turns+1, err) + } + release() + ledger.total += measuredBuilderTurn + } + if turns != 27 { + t.Fatalf("the default budget %d admits %d builder turns of %d tokens a day; want 27", + cfg.DailyTokenBudget, turns, measuredBuilderTurn) + } +} + // TestBudgetGuard_BudgetZero_CheckDisabled: DailyTokenBudget=0 is the explicit opt-out — the // ledger must never be consulted. func TestBudgetGuard_BudgetZero_CheckDisabled(t *testing.T) { diff --git a/nexus-platform/internal/config/ai_config.go b/nexus-platform/internal/config/ai_config.go index b1c514d..90b8c36 100644 --- a/nexus-platform/internal/config/ai_config.go +++ b/nexus-platform/internal/config/ai_config.go @@ -37,7 +37,11 @@ const ( defaultAIRateLimitMaxTurns = 10 defaultAIRateLimitWindowSec = 60 defaultAIMaxConcurrent = 2 - defaultAIDailyTokenBudget = 200000 + // defaultAIDailyTokenBudget is in whole-input tokens (prompt_tokens counts cache reads and writes, and the + // streamed calls report them: issue #30 item 8). One builder turn that makes an app measured 115,084 + // (evals/builder-bench/report.md), so 3M admits 27 a day per tenant, the day 200,000 admitted when a turn + // recorded only its uncached ~8,000. Pinned by TestBudgetGuard_DefaultBudgetAdmitsADayOfBuilderTurns. + defaultAIDailyTokenBudget = 3000000 // ⌘K assistant is every-user (unlike the admin-only builder), so it gets a PER-USER BURST cap // on its share of the per-tenant turn window (defaultAIRateLimitMaxTurns, SHARED by all users // of the tenant + the admin builder). The default is deliberately set BELOW that tenant cap so @@ -88,7 +92,8 @@ type AIConfig struct { MaxConcurrent int // DailyTokenBudget caps SUM(prompt_tokens+output_tokens) per tenant per UTC day, read from // the sys_ai_runs ledger. prompt_tokens includes the prompt-cache reads and writes at full weight - // (issue #30 item 8), so size it for the whole input a turn sends, not the uncached remainder. 0 disables the budget check (EXPLICIT opt-out — the default is + // (issue #30 item 8), so size it for the whole input a turn sends, not the uncached remainder (one + // builder turn ≈ 115,000). 0 disables the budget check (EXPLICIT opt-out — the default is // always bounded; a negative or unparsable env value degrades to the default, not to 0). DailyTokenBudget int64 From b9f3a4751e758e918fb41298dff92a651c5e59ae Mon Sep 17 00:00:00 2001 From: doxuta Date: Tue, 27 Oct 2026 15:07:51 +0700 Subject: [PATCH 6/7] fix(ai): a stream without its finish or its usage is a gateway error; an index-less tool call opens on a new id [DONE] alone no longer finishes an answer (partial tool arguments were accepted), and a stream that closes after the finish with neither its usage nor [DONE] is a 502 instead of an answer that escapes the budget. A proxy that streams parallel tool calls without an index gets one call per id instead of one merged call. .env.example says what happens with a gateway that rejects stream_options. Co-Authored-By: Claude Opus 5.5 --- nexus-platform/.env.example | 3 ++- nexus-platform/internal/ai/gateway.go | 23 ++++++++++++------- .../internal/ai/gateway_stream_test.go | 19 +++++++++++++++ 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/nexus-platform/.env.example b/nexus-platform/.env.example index 8ee0173..d52e7f4 100644 --- a/nexus-platform/.env.example +++ b/nexus-platform/.env.example @@ -37,7 +37,8 @@ REDIS_URL=redis://127.0.0.1:6379 #AI_ENABLED=true # Any OpenAI-compatible /v1/chat/completions gateway works (LiteLLM, vLLM, Ollama, a hosted endpoint). # Calls are streamed with stream_options.include_usage, so the ledger gets the whole input; a gateway -# that ignores stream and answers one JSON body works too. +# that ignores stream and answers one JSON body works too. One that rejects stream_options answers 400 +# and every call fails (502 AI_GATEWAY_ERROR, the server log quotes the gateway): put LiteLLM in front. #AI_GATEWAY_BASE_URL=http://127.0.0.1:4000/v1 # Put the real key in .env only (never commit it). The model id must be one the gateway lists (GET /v1/models): #AI_GATEWAY_API_KEY= diff --git a/nexus-platform/internal/ai/gateway.go b/nexus-platform/internal/ai/gateway.go index 8594868..422d15d 100644 --- a/nexus-platform/internal/ai/gateway.go +++ b/nexus-platform/internal/ai/gateway.go @@ -351,10 +351,13 @@ type streamChunk struct { // foldStream folds a streamed completion into the ChatResponse a non-streamed one gives: the text and the // refusal text joined; each tool call's arguments joined by its index, "{}" when none came, its id, type and -// name taken as sent (9router repeats the id on every piece); the last finish reason and stop reasons; and -// the last usage the stream sent (OpenAI sends it once, after the finish). [DONE], a finish reason or a -// top-level stop reason ends the answer (9router sends no [DONE]); a stream that stops before one, or that -// sends an error event, is an error: its partial tool calls are not an answer. +// name taken as sent (9router repeats the id on every piece; a proxy that leaves the index out gets a new +// call for each new id); the last finish reason and stop reasons; and the last usage the stream sent (OpenAI +// sends it once, after the finish). A finish reason or a top-level stop reason finishes the answer, and +// [DONE] or the usage closes the stream (9router sends no [DONE], its usage rides on the finish chunk). A +// stream that stops before the finish, or after it with neither (the usage lost), or that sends an error +// event, is an error: a partial tool call is not an answer, and an answer without its usage escapes the +// budget. // // ponytail: one "data:" line is one event; a gateway that splits an event over several data lines (the // SSE spec allows it, no OpenAI-compatible gateway does) is read as broken JSON, a 502. @@ -363,7 +366,8 @@ func foldStream(body []byte) (ChatResponse, error) { out ChatResponse choice *ChatChoice slot = map[int]int{} // a tool call's stream index → its place in the message - done bool + done bool // [DONE] came + usage bool // a usage came ) for _, line := range strings.Split(string(body), "\n") { data, ok := strings.CutPrefix(strings.TrimRight(line, "\r"), "data:") @@ -382,7 +386,7 @@ func foldStream(body []byte) (ChatResponse, error) { return ChatResponse{}, fmt.Errorf("ai.foldStream: error event mid-stream: %s", c.Error) } if c.Usage != nil { - out.Usage = *c.Usage + out.Usage, usage = *c.Usage, true } if c.StopReason != nil { out.StopReason = c.StopReason @@ -401,7 +405,7 @@ func foldStream(body []byte) (ChatResponse, error) { choice.Message.Refusal += ch.Delta.Refusal for _, tc := range ch.Delta.ToolCalls { i, seen := slot[tc.Index] - if !seen { + if !seen || tc.ID != "" && choice.Message.ToolCalls[i].ID != "" && tc.ID != choice.Message.ToolCalls[i].ID { i = len(choice.Message.ToolCalls) slot[tc.Index] = i choice.Message.ToolCalls = append(choice.Message.ToolCalls, ToolCall{Type: "function"}) @@ -426,9 +430,12 @@ func foldStream(body []byte) (ChatResponse, error) { } } } - if !done && out.StopReason == nil && (choice == nil || choice.FinishReason == "") { + if out.StopReason == nil && (choice == nil || choice.FinishReason == "") { return ChatResponse{}, errors.New("ai.foldStream: the stream ended before the answer finished") } + if !done && !usage { + return ChatResponse{}, errors.New("ai.foldStream: the stream ended after the answer, before its usage or [DONE]") + } if choice != nil { // A call without arguments streams no piece of them; a whole answer sends "{}", and the gateway // refuses "" when the call is sent back (9router: "tool_use.input: Input should be an object"). diff --git a/nexus-platform/internal/ai/gateway_stream_test.go b/nexus-platform/internal/ai/gateway_stream_test.go index 7e168ba..a9521a7 100644 --- a/nexus-platform/internal/ai/gateway_stream_test.go +++ b/nexus-platform/internal/ai/gateway_stream_test.go @@ -97,6 +97,16 @@ func TestGateway_Complete_FoldsTheStream(t *testing.T) { "data: " + `{"id":"chatcmpl-msg_011","object":"chat.completion.chunk","created":1793086291,"model":"claude-opus-5-5","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":18836,"completion_tokens":4,"total_tokens":18840}}` + "\n\n", ChatResponse{Choices: []ChatChoice{{Message: ChatMessage{Role: "assistant", Content: "ok"}, FinishReason: "stop"}}, Usage: Usage{PromptTokens: 18836, CompletionTokens: 4, TotalTokens: 18840}}}, + {"parallel tool calls streamed without an index: a new id opens a new call", + // Some OpenAI-compatible proxies leave the index out; every piece then reads index 0. + sse(`{"choices":[{"index":0,"delta":{"role":"assistant","tool_calls":[{"id":"c1","type":"function","function":{"name":"a","arguments":"{\"x\":"}}]},"finish_reason":null}]}`, + `{"choices":[{"index":0,"delta":{"tool_calls":[{"function":{"arguments":"1}"}}]},"finish_reason":null}]}`, + `{"choices":[{"index":0,"delta":{"tool_calls":[{"id":"c2","type":"function","function":{"name":"b","arguments":"{\"y\":2}"}}]},"finish_reason":null}]}`, + `{"choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]}`), + ChatResponse{Choices: []ChatChoice{{Message: ChatMessage{Role: "assistant", ToolCalls: []ToolCall{ + {ID: "c1", Type: "function", Function: ToolCallFunc{Name: "a", Arguments: `{"x":1}`}}, + {ID: "c2", Type: "function", Function: ToolCallFunc{Name: "b", Arguments: `{"y":2}`}}, + }}, FinishReason: "tool_calls"}}}}, {"comments, event names and ids between the chunks are not data", ": keep-alive\n\nevent: message\nid: 1\ndata: " + `{"choices":[{"index":0,"delta":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}` + "\n\n: bye\n\ndata: [DONE]\n\n", ChatResponse{Choices: []ChatChoice{{Message: ChatMessage{Role: "assistant", Content: "ok"}, FinishReason: "stop"}}}}, @@ -209,6 +219,15 @@ func TestGateway_Complete_StreamFailures(t *testing.T) { w.Header().Set("Content-Type", "text/event-stream") _, _ = w.Write([]byte("data: " + partial + "\n\n")) }, ErrGateway, "ended before"}, + {"[DONE] before the answer finished", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte("data: " + partial + "\n\ndata: [DONE]\n\n")) + }, ErrGateway, "ended before"}, + {"the stream closes after the finish, before the usage and [DONE]", func(w http.ResponseWriter, r *http.Request) { + // OpenAI sends the usage after the finish, then [DONE]; a stream that stops in between lost the usage. + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte("data: " + `{"choices":[{"index":0,"delta":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}` + "\n\n")) + }, ErrGateway, "before its usage"}, {"a chunk that is not JSON", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/event-stream") _, _ = w.Write([]byte("data: " + partial + "\n\ndata: {not json\n\n")) From dfca320887f7bfe4adfcec0ce01a00f15a5077aa Mon Sep 17 00:00:00 2001 From: doxuta Date: Tue, 27 Oct 2026 15:07:51 +0700 Subject: [PATCH 7/7] test(ai): the read-only pin names PacksOf and AggregateScalar and stops at the logger #70 gave toolExecutor a *zap.Logger and AccessProbes two probes (PacksOf, AggregateScalar) on a base without #68's read-only pin, so main fails the pin since both landed. Both probes are SELECTs and are named as reads; the logger is a listed sink the walk does not enter, and the catch test proves a zapcore.Core reached another way is still reported. Co-Authored-By: Claude Opus 5.5 --- .../internal/ai/tools_readonly_pin_test.go | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/nexus-platform/internal/ai/tools_readonly_pin_test.go b/nexus-platform/internal/ai/tools_readonly_pin_test.go index daaca63..28e5b82 100644 --- a/nexus-platform/internal/ai/tools_readonly_pin_test.go +++ b/nexus-platform/internal/ai/tools_readonly_pin_test.go @@ -12,6 +12,9 @@ import ( "slices" "strings" "testing" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) // Issue #22 item 3: MCP (internal/mcpserver) and the in-app builder run every tool through toolExecutor, @@ -21,7 +24,7 @@ import ( // readVerbs are the method names an executor dependency may have. A new method fails the pin until a // human reads it and adds it here: Get*/List*/Load*/Find* read; the rest are named one by one. -var readVerbs = regexp.MustCompile(`^(Get|List|Load|Find)[A-Z]|^Get$|^(Authorize|HeadCutoff|ColumnExists|RefEntity|AccessProbes|PrivilegedHolders)$`) +var readVerbs = regexp.MustCompile(`^(Get|List|Load|Find)[A-Z]|^Get$|^(Authorize|HeadCutoff|ColumnExists|RefEntity|AccessProbes|PrivilegedHolders|PacksOf|AggregateScalar)$`) // writeWords may not appear anywhere in a read method's name: GetOrCreate, FindOrCreate or ListAndPurge // start like a read and then write. @@ -39,6 +42,12 @@ func isReadMethod(name string) bool { return !slices.ContainsFunc(camelWords.FindAllString(name, -1), func(w string) bool { return slices.Contains(writeWords, w) }) } +// sinks are types the walk does not enter, each with the reason a human read it: they write, but never to +// tenant data, the draft or the model. +var sinks = map[reflect.Type]string{ + reflect.TypeOf(zap.Logger{}): "the server log: warn writes a line there when a best-effort read fails", +} + // executorRoots are the types the executor reaches: its own fields, and the probes it gets by // type-asserting its reader to AccessProbeCarrier (the only interface it asserts to, pinned below). var executorRoots = []reflect.Type{ @@ -67,14 +76,16 @@ func TestToolExecutor_ReadOnlyPinCatchesAWriter(t *testing.T) { reader MetadataReader probes *struct{ Store writer } hook func() + log *zap.Logger // a sink: not entered + core zapcore.Core // what the logger holds, reached another way: entered } got := strings.Join(readOnlyViolations([]reflect.Type{reflect.TypeOf(withWriter{})}), "\n") - for _, want := range []string{"CreateEntity", "GetOrCreateEntity", "FindOrCreateRole", "ListAndPurgeDrafts", "func"} { + for _, want := range []string{"CreateEntity", "GetOrCreateEntity", "FindOrCreateRole", "ListAndPurgeDrafts", "func", "core (zapcore.Core) has Write"} { if !strings.Contains(got, want) { t.Errorf("the pin let %q through: %s", want, got) } } - if strings.Contains(got, "GetEntityByName") || strings.Contains(got, "ListFieldsIncludingDeleted") { + if strings.Contains(got, "GetEntityByName") || strings.Contains(got, "ListFieldsIncludingDeleted") || strings.Contains(got, ".log") { t.Errorf("a read method was reported: %s", got) } } @@ -205,6 +216,9 @@ func readOnlyViolations(roots []reflect.Type) []string { return } seen[t] = true + if sinks[t] != "" { + return + } switch t.Kind() { case reflect.Interface: for i := range t.NumMethod() {