From b0d29b012183bf9779ffdaa4209fc261f4331b2b Mon Sep 17 00:00:00 2001 From: James Burton Date: Mon, 15 Jun 2026 14:17:39 +0100 Subject: [PATCH 1/2] server: Anthropic-compatible Messages API (/v1/messages) (#325) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an Anthropic Messages API layer to DotLLM.Server alongside the existing OpenAI surface, so clients written for the `anthropic` SDKs can talk to dotLLM unchanged. Purely additive — the engine, chat template, sampler and tool-calling pipeline are reused verbatim; only the wire format differs. Endpoints: - POST /v1/messages — non-streaming (JSON) and streaming (named SSE events: message_start, content_block_start/_delta/_stop, message_delta, message_stop). - POST /v1/messages/count_tokens — { "input_tokens": N }. Translation (AnthropicConverter): - Top-level system (string or text-block array) -> leading system message. - String-or-block message content; text/tool_use/tool_result blocks mapped to ChatMessage/ToolCall and tool-role messages keyed by tool_use_id. - tools/input_schema -> ToolDefinition; tool_choice auto/any/none/tool. - FinishReason -> stop_reason (end_turn/max_tokens/stop_sequence/tool_use). - Anthropic error envelope; AOT-clean source-gen DTOs in ServerJsonContext. Tests: 26 unit tests (message flattening, tool_choice, stop_reason, tool_use blocks, validation, response/error serialization shape). Docs: docs/ANTHROPIC_API.md + SERVER.md/ROADMAP.md/README.md/CLAUDE.md sync. Closes #325 Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 1 + README.md | 2 + docs/ANTHROPIC_API.md | 164 +++++++ docs/ROADMAP.md | 1 + docs/SERVER.md | 16 + src/DotLLM.Server/AnthropicConverter.cs | 285 ++++++++++++ src/DotLLM.Server/EndpointExtensions.cs | 1 + .../Endpoints/MessagesEndpoint.cs | 419 ++++++++++++++++++ .../Models/AnthropicMessagesModels.cs | 301 +++++++++++++ src/DotLLM.Server/ServerJsonContext.cs | 11 + .../Server/AnthropicConverterTests.cs | 294 ++++++++++++ 11 files changed, 1495 insertions(+) create mode 100644 docs/ANTHROPIC_API.md create mode 100644 src/DotLLM.Server/AnthropicConverter.cs create mode 100644 src/DotLLM.Server/Endpoints/MessagesEndpoint.cs create mode 100644 src/DotLLM.Server/Models/AnthropicMessagesModels.cs create mode 100644 tests/DotLLM.Tests.Unit/Server/AnthropicConverterTests.cs diff --git a/CLAUDE.md b/CLAUDE.md index 3853825b..01e4568c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -146,6 +146,7 @@ dotLLM/ | Diagnostics & interpretability | [docs/DIAGNOSTICS.md](docs/DIAGNOSTICS.md) | Hooks, logit lens, SAE | | Telemetry & observability | [docs/TELEMETRY.md](docs/TELEMETRY.md) | Metrics, request tracing | | Server & API | [docs/SERVER.md](docs/SERVER.md) | Endpoints, rate limiting, warm-up | +| Anthropic Messages API | [docs/ANTHROPIC_API.md](docs/ANTHROPIC_API.md) | `/v1/messages`, content blocks, streaming events, mapping | | Warm-up | [docs/WARMUP.md](docs/WARMUP.md) | Startup warm-up, JIT compilation, CUDA warm-up | | GPU inference | [docs/GPU.md](docs/GPU.md) | GPU forward pass, weight loading, KV-cache, CLI | | CUDA backend | [docs/CUDA.md](docs/CUDA.md) | PTX architecture, P/Invoke, kernel conventions, build | diff --git a/README.md b/README.md index 7092f8a8..d27feb7b 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ dotLLM is a ground-up LLM inference engine for .NET — not a wrapper around lla ### Serving - **OpenAI-compatible API** — `/v1/chat/completions`, `/v1/completions`, tool calling, streaming via ASP.NET +- **Anthropic-compatible API** — `/v1/messages` (+ `/v1/messages/count_tokens`), event-based SSE streaming, served alongside the OpenAI surface - **Paged KV-cache** — PagedAttention with block-level allocation, prefix caching, and copy-on-write - **Speculative decoding** — draft-verify-accept with KV-cache rollback (greedy mode today; non-greedy planned — see issue #121) - **Structured output** — FSM/PDA-based constrained decoding guaranteeing valid JSON, JSON Schema, regex, and grammar @@ -672,6 +673,7 @@ Both modes transparently reuse the embedded chat UI assets if `serveUi: true`. T ## News +- **2026-06** — **Anthropic-compatible Messages API** — `/v1/messages` (non-streaming + event-based SSE: `message_start`, `content_block_start`/`_delta`/`_stop`, `message_delta`, `message_stop`) and `/v1/messages/count_tokens`, served alongside the OpenAI surface. Top-level `system` (string or text-block array), string-or-block message `content`, `tool_use`/`tool_result` content blocks, `tool_choice` (`auto`/`any`/`none`/`tool`), and `FinishReason → stop_reason` mapping (`end_turn`/`max_tokens`/`stop_sequence`/`tool_use`). Reuses the shared engine, chat-template, sampler and tool-calling pipeline — only the wire format differs (`MessagesEndpoint` + `AnthropicConverter`). 26 unit tests; see `docs/ANTHROPIC_API.md` ([#325](https://github.com/kkokosa/dotLLM/issues/325)) - **2026-04** — **First public release (v0.1.0-preview.1)** — dotLLM goes public. [NuGet packages](#nuget-packages) for all 10 libraries + `DotLLM.Cli` as a global `dotnet tool`. Self-contained single-file downloads for Windows / Linux / macOS (Apple Silicon) and experimental Native AOT builds for Linux / Windows attached to every [GitHub Release](https://github.com/kkokosa/dotLLM/releases). Companion website at [dotllm.dev](https://dotllm.dev/) ([#119](https://github.com/kkokosa/dotLLM/issues/119)) - **2026-04** — **Wave 7**: CPU performance cleanup pass — `TopKSampler` replaces full `Array.Sort` with a hand-rolled size-K min-heap (`O(N log K)`, stack-resident scratch); `JsonSchemaConstraint` adds first-char bucketing to skip the ~160 MB of struct clones per mask build when the tracker rejects most leading characters, plus LRU eviction instead of the previous full-flush cache overflow; `Dequantize.Q5_0` gains an AVX2 path matching Q8_0's throughput (reuses `MatMulQ5_0.ExtractQ5HighBits` / `vpshufb` bit-extraction); `BpeTokenizer` pre-splits special tokens via the existing `Trie.TryMatchLongest` instead of the O(n × m) linear scan; `ComputeThreadPool` now pins the caller (inference) thread to the first candidate P-core on first `Dispatch`, eliminating the hybrid-CPU stall where pinned P-core workers idled at the barrier waiting for an E-core caller. New BenchmarkDotNet suites for TopK sampling, schema mask build, and special-token encode ([#109](https://github.com/kkokosa/dotLLM/issues/109)) - **2026-04** — **Phase 7 begins**: Logprobs — OpenAI-compatible `logprobs: true` + `top_logprobs: N` (0-20) on `/v1/chat/completions` and `/v1/completions`. Per-token log-softmax captured before sampling, returned in both streaming SSE chunks and non-streaming responses. Chat UI gains opt-in logprobs visualization: color-coded token confidence (green/lime/yellow/orange/red), hover tooltips with top-K alternatives and probabilities, diagnostic cues for low confidence, ambiguity, and sampling effect. `DotLLM.Sample.Logprobs` console sample with ANSI-colored output ([#101](https://github.com/kkokosa/dotLLM/issues/101)) diff --git a/docs/ANTHROPIC_API.md b/docs/ANTHROPIC_API.md new file mode 100644 index 00000000..f1d97045 --- /dev/null +++ b/docs/ANTHROPIC_API.md @@ -0,0 +1,164 @@ +# Anthropic Messages API — dotLLM + +dotLLM's server exposes an **Anthropic-compatible Messages API** alongside the +OpenAI-compatible surface ([SERVER.md](SERVER.md)). Clients and SDKs written for +the Anthropic Messages API (`anthropic` Python/TypeScript SDKs, anything that +targets `POST /v1/messages`) can point at a running dotLLM server unchanged. + +The engine, tokenizer, chat-template, sampler and tool-calling pipeline are +shared verbatim with the OpenAI endpoints — this layer only reshapes the wire +format. Implementation: `MessagesEndpoint`, `AnthropicConverter`, and the +`Anthropic*` DTOs in `DotLLM.Server`. + +Reference: + +## Endpoints + +### `POST /v1/messages` + +Primary endpoint. Accepts the Anthropic Messages request format; supports both +non-streaming (JSON) and streaming (named SSE events). + +**Request body**: +```json +{ + "model": "llama-3-8b-q4_k_m", + "max_tokens": 256, + "system": "You are helpful.", + "messages": [ + {"role": "user", "content": "Hello!"}, + {"role": "assistant", "content": [{"type": "text", "text": "Hi!"}]}, + {"role": "user", "content": "What's the weather?"} + ], + "temperature": 0.7, + "top_p": 0.9, + "top_k": 40, + "stop_sequences": ["\n\nHuman:"], + "tools": [ + {"name": "get_weather", "description": "Get weather", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}}} + ], + "tool_choice": {"type": "auto"}, + "stream": false, + "lora_adapter": "customer-support" +} +``` + +- `max_tokens` is **required** (per the Anthropic spec). Missing/`<= 0` → `400`. +- `system` is a top-level string **or** an array of `{"type":"text","text":"..."}` + blocks; it becomes a leading `system` message in the chat template. +- Each message `content` is a string **or** an array of content blocks + (`text`, `tool_use`, `tool_result`). +- `tool_choice`: `{"type":"auto"}`, `{"type":"any"}` (→ required), + `{"type":"none"}`, or `{"type":"tool","name":"..."}`. +- `lora_adapter` is a dotLLM extension (parity with the OpenAI surface). +- `image` content blocks are not yet supported (no multimodal pipeline). + +**Response** (non-streaming): +```json +{ + "id": "msg_...", + "type": "message", + "role": "assistant", + "model": "llama-3-8b-q4_k_m", + "content": [{"type": "text", "text": "It's sunny."}], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": {"input_tokens": 15, "output_tokens": 8} +} +``` + +When tool calls are detected, `content` contains `tool_use` blocks and +`stop_reason` is `"tool_use"`: +```json +{ + "content": [ + {"type": "tool_use", "id": "toolu_...", "name": "get_weather", + "input": {"city": "Paris"}} + ], + "stop_reason": "tool_use" +} +``` + +### `POST /v1/messages/count_tokens` + +Returns the prompt token count for a would-be request. Same body as +`/v1/messages` (without requiring `max_tokens`). + +**Response**: `{"input_tokens": 42}` + +## Streaming + +With `"stream": true`, the response is a sequence of **named** SSE events +(`event: \ndata: \n\n`): + +``` +event: message_start +data: {"type":"message_start","message":{"id":"msg_...","type":"message","role":"assistant","model":"...","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":15,"output_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + +event: ping +data: {"type":"ping"} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"It's"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":15,"output_tokens":8}} + +event: message_stop +data: {"type":"message_stop"} +``` + +Tool calls detected during streaming are emitted after the text block closes, as +additional `tool_use` content blocks (`content_block_start` → +`content_block_delta` with `input_json_delta` → `content_block_stop`) at index +`1+`, and `stop_reason` becomes `"tool_use"`. + +## Mapping reference + +| Anthropic field | dotLLM engine | +|-----------------|---------------| +| `system` (string/array) | leading `system` `ChatMessage` | +| message `content` string | `ChatMessage.Content` | +| `text` block | concatenated into `ChatMessage.Content` | +| `tool_use` block (assistant) | `ChatMessage.ToolCalls` (`ToolCall`) | +| `tool_result` block (user) | separate `tool`-role `ChatMessage` keyed by `tool_use_id` | +| `tools[].input_schema` | `ToolDefinition.ParametersSchema` | +| `tool_choice` `auto`/`any`/`none`/`tool` | `ToolChoice.Auto`/`Required`/`None`/`Function` | +| `stop_sequences` | `InferenceOptions.StopSequences` | + +| dotLLM `FinishReason` | Anthropic `stop_reason` | +|-----------------------|-------------------------| +| `Stop` (EOS / template stop) | `end_turn` | +| `Stop` (caller `stop_sequences` matched) | `stop_sequence` (+ `stop_sequence` field) | +| `Length` | `max_tokens` | +| `ToolCalls` | `tool_use` | + +## Errors + +Errors use the Anthropic envelope: +```json +{"type": "error", "error": {"type": "invalid_request_error", "message": "max_tokens: field required"}} +``` + +| Condition | HTTP | `error.type` | +|-----------|------|--------------| +| No model loaded | 503 | `api_error` | +| Empty `messages`, missing/invalid `max_tokens`, bad LoRA name | 400 | `invalid_request_error` | +| Prompt exceeds context window | 400 | `invalid_request_error` | + +## Limitations + +- **Streaming tool calls** are detected post-generation (the engine parses tool + calls from the full output), so `tool_use` blocks are emitted at the end of the + stream rather than incrementally — matching the OpenAI streaming endpoint's + post-hoc detection. +- **`image` / multimodal content blocks** are not supported. +- The same single-request serialization, prompt caching, and validation rules as + the OpenAI endpoints apply (see [SERVER.md](SERVER.md)). diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index b0b9a995..e2fa8700 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -97,6 +97,7 @@ Step 22 (done) ──────► Step 30 (NUMA + Spin-wait) | 41 | **Regex + CFG** :white_check_mark: | `RegexConstraint` (DFA-based) and `GrammarConstraint` (PDA, GBNF-style). | 39 | | 42 | **Tool calling** :white_check_mark: | `IToolCallParser`, chat template tool integration, structured output for function arguments. `finish_reason: "tool_calls"`. Parallel tool calls. | 16, 40 | | 34 | **ASP.NET server** :white_check_mark: | Minimal API endpoints: `/v1/chat/completions`, `/v1/completions`, `/v1/models`, `/v1/embeddings`, `/v1/tokenize`, `/v1/detokenize`. Health + readiness probes. | 16, 17 | +| 34b | **Anthropic Messages API** :white_check_mark: | Anthropic-compatible `/v1/messages` (non-streaming + event-based SSE) and `/v1/messages/count_tokens`, served alongside the OpenAI surface. Top-level `system`, string-or-block content, `tool_use`/`tool_result` blocks, `tool_choice`, `stop_reason` mapping. Reuses the shared engine/chat-template/sampler/tool-calling pipeline — wire format only. See `docs/ANTHROPIC_API.md`. | 34, 42 | | 53 | **Chat UI (`serve` command)** :white_check_mark: | Built-in web chat interface served by the ASP.NET host. `dotllm serve model.gguf` starts the API server and opens a browser to a bundled single-page chat UI. Streaming responses via SSE, model/parameter selection, conversation history. Similar to `ollama serve` + Open WebUI or `llama.cpp --server` built-in UI. | 34 | | 54 | **Simple prompt caching** :white_check_mark: | Hash-based prefix cache for multi-turn conversations. Hash the token prefix, store/reuse the KV-cache state. On cache hit, skip prefill for the matching prefix and only process new tokens. Works with existing `SimpleKvCache` — no paged attention required. LRU eviction with configurable max cached sessions. `--prompt-cache` CLI flag. Dramatically reduces TTFT for multi-turn chat. | 7, 34 | diff --git a/docs/SERVER.md b/docs/SERVER.md index a8ae065f..acfef0c6 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -79,6 +79,22 @@ Not in OpenAI spec but widely expected for prompt engineering and billing estima **Request**: `{"tokens": [9906, 1917], "model": "..."}` **Response**: `{"text": "Hello world"}` +### `POST /v1/messages` (Anthropic-compatible) + +Anthropic Messages API endpoint, served alongside the OpenAI surface so that +`anthropic` SDK clients can talk to dotLLM unchanged. Top-level `system`, +string-or-block message `content`, `max_tokens` (required), `stop_sequences`, +`tools`/`tool_choice`, and event-based streaming SSE (`message_start`, +`content_block_*`, `message_delta`, `message_stop`). Reuses the same engine, +chat-template, sampler and tool-calling pipeline; only the wire format differs. + +### `POST /v1/messages/count_tokens` (Anthropic-compatible) + +Returns `{"input_tokens": N}` for a would-be Messages request. + +See **[ANTHROPIC_API.md](ANTHROPIC_API.md)** for the full request/response shapes, +streaming event sequence, and the engine mapping. + ## response_format Processing The `response_format` field maps to constrained decoding: diff --git a/src/DotLLM.Server/AnthropicConverter.cs b/src/DotLLM.Server/AnthropicConverter.cs new file mode 100644 index 00000000..d71c3cb5 --- /dev/null +++ b/src/DotLLM.Server/AnthropicConverter.cs @@ -0,0 +1,285 @@ +using System.Text; +using System.Text.Json; +using DotLLM.Core.Configuration; +using DotLLM.Engine; +using DotLLM.Server.Models; +using DotLLM.Tokenizers; + +namespace DotLLM.Server; + +/// +/// Converts between Anthropic Messages API DTOs and dotLLM engine types. +/// The engine, chat template, sampler and tool-calling pipeline are shared +/// verbatim with the OpenAI surface — this type only reshapes the wire format. +/// +public static class AnthropicConverter +{ + /// + /// Flattens an Anthropic request (top-level system + per-message + /// content blocks) into the engine's linear list. + /// + /// + /// + /// A top-level system string/array becomes a leading system message. + /// String message content maps 1:1. + /// text blocks are concatenated; tool_use blocks become s; + /// tool_result blocks become separate tool-role messages keyed by tool_use_id. + /// + /// + public static ChatMessage[] ToMessages(AnthropicMessagesRequest request) + { + var result = new List(request.Messages.Length + 1); + + string? systemText = ExtractText(request.System); + if (!string.IsNullOrEmpty(systemText)) + result.Add(new ChatMessage { Role = "system", Content = systemText }); + + foreach (var msg in request.Messages) + AppendMessage(result, msg); + + return result.ToArray(); + } + + private static void AppendMessage(List target, AnthropicMessageDto msg) + { + var content = msg.Content; + + if (content.ValueKind == JsonValueKind.String) + { + target.Add(new ChatMessage { Role = msg.Role, Content = content.GetString() ?? "" }); + return; + } + + if (content.ValueKind != JsonValueKind.Array) + { + target.Add(new ChatMessage { Role = msg.Role, Content = "" }); + return; + } + + var textBuilder = new StringBuilder(); + List? toolCalls = null; + + foreach (var block in content.EnumerateArray()) + { + if (block.ValueKind != JsonValueKind.Object) + continue; + + string? blockType = block.TryGetProperty("type", out var t) ? t.GetString() : null; + switch (blockType) + { + case "text": + if (block.TryGetProperty("text", out var txt) && txt.ValueKind == JsonValueKind.String) + { + if (textBuilder.Length > 0) textBuilder.Append('\n'); + textBuilder.Append(txt.GetString()); + } + break; + + case "tool_use": + string tuId = block.TryGetProperty("id", out var idp) ? idp.GetString() ?? "" : ""; + string tuName = block.TryGetProperty("name", out var np) ? np.GetString() ?? "" : ""; + string tuInput = block.TryGetProperty("input", out var ip) ? ip.GetRawText() : "{}"; + (toolCalls ??= []).Add(new ToolCall(tuId, tuName, tuInput)); + break; + + case "tool_result": + // Emit any pending text/tool_use for this message, then the tool result. + FlushPending(target, msg.Role, textBuilder, ref toolCalls); + string trId = block.TryGetProperty("tool_use_id", out var tup) ? tup.GetString() ?? "" : ""; + target.Add(new ChatMessage + { + Role = "tool", + Content = ExtractToolResultContent(block), + ToolCallId = trId, + }); + break; + } + } + + FlushPending(target, msg.Role, textBuilder, ref toolCalls); + } + + private static void FlushPending( + List target, string role, StringBuilder textBuilder, ref List? toolCalls) + { + if (textBuilder.Length == 0 && toolCalls is null) + return; + + target.Add(new ChatMessage + { + Role = role, + Content = textBuilder.ToString(), + ToolCalls = toolCalls?.ToArray(), + }); + textBuilder.Clear(); + toolCalls = null; + } + + private static string ExtractToolResultContent(JsonElement block) + { + if (!block.TryGetProperty("content", out var c)) + return ""; + if (c.ValueKind == JsonValueKind.String) + return c.GetString() ?? ""; + if (c.ValueKind == JsonValueKind.Array) + return ConcatTextBlocks(c); + return ""; + } + + /// + /// Extracts plain text from an Anthropic system value: a string, or an + /// array of {"type":"text","text":"..."} blocks. Returns null when absent. + /// + public static string? ExtractText(JsonElement? element) + { + if (element is null) + return null; + var e = element.Value; + if (e.ValueKind == JsonValueKind.String) + return e.GetString(); + if (e.ValueKind == JsonValueKind.Array) + return ConcatTextBlocks(e); + return null; + } + + private static string ConcatTextBlocks(JsonElement array) + { + var sb = new StringBuilder(); + foreach (var item in array.EnumerateArray()) + { + if (item.ValueKind == JsonValueKind.Object && + item.TryGetProperty("text", out var t) && t.ValueKind == JsonValueKind.String) + { + if (sb.Length > 0) sb.Append('\n'); + sb.Append(t.GetString()); + } + } + return sb.ToString(); + } + + /// Converts Anthropic tool definitions to engine s. + public static ToolDefinition[]? ToTools(AnthropicToolDto[]? dtos) + { + if (dtos is null) + return null; + var result = new ToolDefinition[dtos.Length]; + for (int i = 0; i < dtos.Length; i++) + { + var d = dtos[i]; + result[i] = new ToolDefinition(d.Name, d.Description ?? "", d.InputSchema?.GetRawText() ?? "{}"); + } + return result; + } + + /// + /// Parses Anthropic tool_choice into the engine : + /// auto→Auto, any→Required, none→None, tool→Function. + /// + public static ToolChoice ParseToolChoice(JsonElement? element) + { + if (element is null || element.Value.ValueKind != JsonValueKind.Object) + return new ToolChoice.Auto(); + + var e = element.Value; + if (!e.TryGetProperty("type", out var typeProp)) + return new ToolChoice.Auto(); + + return typeProp.GetString() switch + { + "any" => new ToolChoice.Required(), + "none" => new ToolChoice.None(), + "tool" when e.TryGetProperty("name", out var n) && n.ValueKind == JsonValueKind.String => + new ToolChoice.Function(n.GetString()!), + _ => new ToolChoice.Auto(), + }; + } + + /// Builds from an Anthropic request. + public static InferenceOptions ToInferenceOptions( + AnthropicMessagesRequest request, IReadOnlyList commonStops, + SamplingDefaults defaults, ThreadingConfig threading) + { + var allStops = new List(commonStops); + if (request.StopSequences is { Length: > 0 }) + { + foreach (var s in request.StopSequences) + if (!string.IsNullOrEmpty(s)) + allStops.Add(s); + } + + return new InferenceOptions + { + Temperature = request.Temperature ?? defaults.Temperature, + TopK = request.TopK ?? defaults.TopK, + TopP = request.TopP ?? defaults.TopP, + MinP = defaults.MinP, + RepetitionPenalty = defaults.RepetitionPenalty, + MaxTokens = request.MaxTokens ?? defaults.MaxTokens, + Seed = defaults.Seed, + StopSequences = allStops, + Threading = threading, + }; + } + + /// + /// Maps an engine to an Anthropic stop_reason. + /// + /// The engine finish reason. + /// + /// True when generation stopped because a caller-supplied stop sequence matched + /// (reported as stop_sequence rather than end_turn). + /// + public static string ToStopReason(FinishReason reason, bool matchedStopSequence) => reason switch + { + FinishReason.ToolCalls => "tool_use", + FinishReason.Length => "max_tokens", + FinishReason.Stop => matchedStopSequence ? "stop_sequence" : "end_turn", + _ => "end_turn", + }; + + /// Converts detected engine tool calls to Anthropic tool_use content blocks. + public static AnthropicContentBlockDto[] ToToolUseBlocks(ToolCall[] toolCalls) + { + var blocks = new AnthropicContentBlockDto[toolCalls.Length]; + for (int i = 0; i < toolCalls.Length; i++) + { + var tc = toolCalls[i]; + blocks[i] = new AnthropicContentBlockDto + { + Type = "tool_use", + Id = string.IsNullOrEmpty(tc.Id) ? GenerateToolUseId() : tc.Id, + Name = tc.FunctionName, + Input = ParseInput(tc.Arguments), + }; + } + return blocks; + } + + /// Parses a tool-call argument JSON string into a JSON object element. + public static JsonElement ParseInput(string? arguments) + { + if (string.IsNullOrWhiteSpace(arguments)) + return EmptyObject(); + try + { + using var doc = JsonDocument.Parse(arguments); + return doc.RootElement.Clone(); + } + catch (JsonException) + { + return EmptyObject(); + } + } + + private static JsonElement EmptyObject() + { + using var doc = JsonDocument.Parse("{}"); + return doc.RootElement.Clone(); + } + + /// Generates an Anthropic-style message id (msg_...). + public static string GenerateMessageId() => $"msg_{Guid.NewGuid():N}"; + + /// Generates an Anthropic-style tool-use block id (toolu_...). + public static string GenerateToolUseId() => $"toolu_{Guid.NewGuid():N}"; +} diff --git a/src/DotLLM.Server/EndpointExtensions.cs b/src/DotLLM.Server/EndpointExtensions.cs index 2e75609f..e0850a45 100644 --- a/src/DotLLM.Server/EndpointExtensions.cs +++ b/src/DotLLM.Server/EndpointExtensions.cs @@ -15,6 +15,7 @@ public static class EndpointExtensions public static WebApplication MapDotLLMEndpoints(this WebApplication app, bool serveUi = false) { ChatCompletionEndpoint.Map(app); + MessagesEndpoint.Map(app); CompletionEndpoint.Map(app); ModelEndpoint.Map(app); TokenizeEndpoint.Map(app); diff --git a/src/DotLLM.Server/Endpoints/MessagesEndpoint.cs b/src/DotLLM.Server/Endpoints/MessagesEndpoint.cs new file mode 100644 index 00000000..94541a5f --- /dev/null +++ b/src/DotLLM.Server/Endpoints/MessagesEndpoint.cs @@ -0,0 +1,419 @@ +using System.Text; +using System.Text.Json; +using DotLLM.Engine; +using DotLLM.Server.Models; +using DotLLM.Tokenizers; + +namespace DotLLM.Server.Endpoints; + +/// +/// Anthropic-compatible Messages API: +/// +/// POST /v1/messages — non-streaming (JSON) and streaming (named SSE events). +/// POST /v1/messages/count_tokens — prompt token count. +/// +/// Reshapes the Anthropic wire format onto the shared engine pipeline that also +/// backs . Reference: https://docs.anthropic.com/en/api/messages +/// +public static class MessagesEndpoint +{ + private static readonly string[] CommonStopSequences = + ["<|im_end|>", "<|eot_id|>", "<|eom_id|>", "<|end|>", "", ""]; + + public static void Map(WebApplication app) + { + app.MapPost("/v1/messages", HandleAsync); + app.MapPost("/v1/messages/count_tokens", HandleCountTokensAsync); + } + + private static async Task HandleAsync( + AnthropicMessagesRequest request, + ServerState state, + HttpContext httpContext) + { + if (!state.IsReady || state.Generator is null || state.ChatTemplate is null) + { + await WriteErrorAsync(httpContext, 503, "api_error", "No model loaded"); + return; + } + + var validationError = ValidateRequest(request, requireMaxTokens: true); + if (validationError is not null) + { + await WriteErrorAsync(httpContext, 400, "invalid_request_error", validationError); + return; + } + + var ct = httpContext.RequestAborted; + var messageId = AnthropicConverter.GenerateMessageId(); + var modelId = request.Model ?? state.Options.ModelId; + var generator = state.Generator; + + var messages = AnthropicConverter.ToMessages(request); + var tools = AnthropicConverter.ToTools(request.Tools); + + var templateOptions = new ChatTemplateOptions + { + AddGenerationPrompt = true, + Tools = tools, + }; + string prompt = state.ChatTemplate.Apply(messages, templateOptions); + + int maxTokens = request.MaxTokens ?? state.SamplingDefaults.MaxTokens; + var promptError = RequestValidator.ValidatePromptLength( + prompt, state.Tokenizer!, state.Config!.MaxSequenceLength, + maxTokens, out int effectiveMaxTokens, out int promptTokenCount); + if (promptError is not null) + { + await WriteErrorAsync(httpContext, 400, "invalid_request_error", promptError); + return; + } + + var options = AnthropicConverter.ToInferenceOptions(request, CommonStopSequences, + state.SamplingDefaults, + new DotLLM.Core.Configuration.ThreadingConfig(state.Options.Threads, state.Options.DecodeThreads)); + options = options with { MaxTokens = effectiveMaxTokens }; + + if (request.Stream) + await HandleStreamingAsync(request, generator, state, httpContext, prompt, options, + messageId, modelId, tools, promptTokenCount, ct); + else + await HandleNonStreamingAsync(request, generator, state, httpContext, prompt, options, + messageId, modelId, tools, ct); + } + + private static async Task HandleNonStreamingAsync( + AnthropicMessagesRequest request, + TextGenerator generator, + ServerState state, + HttpContext httpContext, + string prompt, + DotLLM.Core.Configuration.InferenceOptions options, + string messageId, string modelId, + ToolDefinition[]? tools, + CancellationToken ct) + { + InferenceResponse? result = null; + await state.ExecuteAsync(async () => + { + result = generator.Generate(prompt, options); + }, ct); + + string text = result!.Text; + ToolCall[]? toolCalls = null; + var finishReason = result.FinishReason; + + if (state.ToolCallParser is not null && tools is { Length: > 0 }) + { + var enriched = ToolCallDetector.DetectToolCalls(result, state.ToolCallParser); + text = enriched.Text; + toolCalls = enriched.ToolCalls; + finishReason = enriched.FinishReason; + } + + // Determine whether a caller-supplied stop sequence ended generation, and strip it. + bool matchedStopSequence = false; + if (finishReason == FinishReason.Stop) + text = StripAndDetectStopSequence(text, request.StopSequences, options.StopSequences, + out matchedStopSequence, out _); + + AnthropicContentBlockDto[] content; + string stopReason; + if (toolCalls is { Length: > 0 }) + { + content = AnthropicConverter.ToToolUseBlocks(toolCalls); + stopReason = "tool_use"; + } + else + { + content = [new AnthropicContentBlockDto { Type = "text", Text = text }]; + stopReason = AnthropicConverter.ToStopReason(finishReason, matchedStopSequence); + } + + string? stopSequence = stopReason == "stop_sequence" + ? MatchStopSequence(result.Text, request.StopSequences) + : null; + + var response = new AnthropicMessageResponse + { + Id = messageId, + Model = modelId, + Content = content, + StopReason = stopReason, + StopSequence = stopSequence, + Usage = new AnthropicUsageDto + { + InputTokens = result.PromptTokenCount, + OutputTokens = result.GeneratedTokenCount, + }, + }; + + httpContext.Response.ContentType = "application/json"; + await JsonSerializer.SerializeAsync(httpContext.Response.Body, response, + ServerJsonContext.Default.AnthropicMessageResponse, ct); + } + + private static async Task HandleStreamingAsync( + AnthropicMessagesRequest request, + TextGenerator generator, + ServerState state, + HttpContext httpContext, + string prompt, + DotLLM.Core.Configuration.InferenceOptions options, + string messageId, string modelId, + ToolDefinition[]? tools, + int promptTokenCount, + CancellationToken ct) + { + httpContext.Response.ContentType = "text/event-stream"; + httpContext.Response.Headers.CacheControl = "no-cache"; + httpContext.Response.Headers.Connection = "keep-alive"; + + // message_start — input_tokens known up front from the prompt. + var startMessage = new AnthropicMessageResponse + { + Id = messageId, + Model = modelId, + Content = [], + StopReason = null, + StopSequence = null, + Usage = new AnthropicUsageDto { InputTokens = promptTokenCount, OutputTokens = 0 }, + }; + await WriteEventAsync(httpContext, "message_start", + new AnthropicMessageStartEvent { Message = startMessage }, + ServerJsonContext.Default.AnthropicMessageStartEvent, ct); + + // Text content block opens at index 0. + await WriteEventAsync(httpContext, "content_block_start", + new AnthropicContentBlockStartEvent + { + Index = 0, + ContentBlock = new AnthropicContentBlockDto { Type = "text", Text = "" }, + }, + ServerJsonContext.Default.AnthropicContentBlockStartEvent, ct); + await WriteEventAsync(httpContext, "ping", new AnthropicPingEvent(), + ServerJsonContext.Default.AnthropicPingEvent, ct); + + var sb = new StringBuilder(); + FinishReason finishReason = FinishReason.Length; + int completionTokens = 0; + + await state.ExecuteAsync(async () => + { + await foreach (var token in generator.GenerateStreamingTokensAsync(prompt, options, ct)) + { + if (token.Text.Length > 0) + { + completionTokens++; + sb.Append(token.Text); + await WriteEventAsync(httpContext, "content_block_delta", + new AnthropicContentBlockDeltaEvent + { + Index = 0, + Delta = new AnthropicStreamDeltaDto { Type = "text_delta", Text = token.Text }, + }, + ServerJsonContext.Default.AnthropicContentBlockDeltaEvent, ct); + } + + if (token.FinishReason.HasValue) + finishReason = token.FinishReason.Value; + } + }, ct); + + // Close the text block. + await WriteEventAsync(httpContext, "content_block_stop", + new AnthropicContentBlockStopEvent { Index = 0 }, + ServerJsonContext.Default.AnthropicContentBlockStopEvent, ct); + + // Post-generation tool-call detection (mirrors the OpenAI streaming endpoint). + string text = sb.ToString(); + ToolCall[]? toolCalls = null; + if (state.ToolCallParser is not null && tools is { Length: > 0 }) + { + toolCalls = state.ToolCallParser.TryParse(text); + if (toolCalls is { Length: > 0 }) + finishReason = FinishReason.ToolCalls; + } + + // Emit detected tool calls as tool_use blocks after the text block. + if (toolCalls is { Length: > 0 }) + { + var blocks = AnthropicConverter.ToToolUseBlocks(toolCalls); + for (int i = 0; i < blocks.Length; i++) + { + int index = i + 1; + var block = blocks[i]; + await WriteEventAsync(httpContext, "content_block_start", + new AnthropicContentBlockStartEvent + { + Index = index, + ContentBlock = new AnthropicContentBlockDto + { + Type = "tool_use", + Id = block.Id, + Name = block.Name, + Input = AnthropicConverter.ParseInput("{}"), + }, + }, + ServerJsonContext.Default.AnthropicContentBlockStartEvent, ct); + await WriteEventAsync(httpContext, "content_block_delta", + new AnthropicContentBlockDeltaEvent + { + Index = index, + Delta = new AnthropicStreamDeltaDto + { + Type = "input_json_delta", + PartialJson = block.Input?.GetRawText() ?? "{}", + }, + }, + ServerJsonContext.Default.AnthropicContentBlockDeltaEvent, ct); + await WriteEventAsync(httpContext, "content_block_stop", + new AnthropicContentBlockStopEvent { Index = index }, + ServerJsonContext.Default.AnthropicContentBlockStopEvent, ct); + } + } + + bool matchedStopSequence = false; + if (finishReason == FinishReason.Stop) + matchedStopSequence = MatchStopSequence(text, request.StopSequences) is not null; + + string stopReason = AnthropicConverter.ToStopReason(finishReason, matchedStopSequence); + string? stopSequence = stopReason == "stop_sequence" + ? MatchStopSequence(text, request.StopSequences) + : null; + + await WriteEventAsync(httpContext, "message_delta", + new AnthropicMessageDeltaEvent + { + Delta = new AnthropicMessageDeltaBody { StopReason = stopReason, StopSequence = stopSequence }, + Usage = new AnthropicUsageDto { InputTokens = promptTokenCount, OutputTokens = completionTokens }, + }, + ServerJsonContext.Default.AnthropicMessageDeltaEvent, ct); + + await WriteEventAsync(httpContext, "message_stop", new AnthropicMessageStopEvent(), + ServerJsonContext.Default.AnthropicMessageStopEvent, ct); + await httpContext.Response.Body.FlushAsync(ct); + } + + private static async Task HandleCountTokensAsync( + AnthropicMessagesRequest request, + ServerState state, + HttpContext httpContext) + { + if (!state.IsReady || state.ChatTemplate is null || state.Tokenizer is null) + { + await WriteErrorAsync(httpContext, 503, "api_error", "No model loaded"); + return; + } + + var validationError = ValidateRequest(request, requireMaxTokens: false); + if (validationError is not null) + { + await WriteErrorAsync(httpContext, 400, "invalid_request_error", validationError); + return; + } + + var ct = httpContext.RequestAborted; + var messages = AnthropicConverter.ToMessages(request); + var tools = AnthropicConverter.ToTools(request.Tools); + string prompt = state.ChatTemplate.Apply(messages, + new ChatTemplateOptions { AddGenerationPrompt = true, Tools = tools }); + + int count = state.Tokenizer.CountTokens(prompt); + + httpContext.Response.ContentType = "application/json"; + await JsonSerializer.SerializeAsync(httpContext.Response.Body, + new AnthropicCountTokensResponse { InputTokens = count }, + ServerJsonContext.Default.AnthropicCountTokensResponse, ct); + } + + /// Validates the structural invariants of an Anthropic request. + internal static string? ValidateRequest(AnthropicMessagesRequest request, bool requireMaxTokens) + { + if (request.Messages is null || request.Messages.Length == 0) + return "messages: at least one message is required"; + + if (request.Messages.Length > RequestValidator.MaxMessages) + return $"messages: array exceeds maximum of {RequestValidator.MaxMessages}"; + + if (requireMaxTokens) + { + if (!request.MaxTokens.HasValue) + return "max_tokens: field required"; + if (request.MaxTokens.Value <= 0) + return "max_tokens: must be a positive integer"; + } + + return null; + } + + /// + /// Returns the request stop sequence that is a suffix of , + /// or null if none matches. + /// + private static string? MatchStopSequence(string text, string[]? stopSequences) + { + if (stopSequences is null) + return null; + foreach (var seq in stopSequences) + { + if (!string.IsNullOrEmpty(seq) && text.EndsWith(seq, StringComparison.Ordinal)) + return seq; + } + return null; + } + + /// + /// Strips a trailing stop-sequence suffix from and reports + /// whether a caller-supplied stop sequence matched. + /// + private static string StripAndDetectStopSequence( + string text, string[]? requestStops, IReadOnlyList allStops, + out bool matchedRequestStop, out string? matched) + { + matchedRequestStop = false; + matched = null; + + // Caller-supplied stop sequences are reported as "stop_sequence". + string? requestMatch = MatchStopSequence(text, requestStops); + if (requestMatch is not null) + { + matchedRequestStop = true; + matched = requestMatch; + return text[..^requestMatch.Length]; + } + + // Built-in/template stop sequences are stripped but reported as "end_turn". + foreach (var seq in allStops) + { + if (text.EndsWith(seq, StringComparison.Ordinal)) + { + matched = seq; + return text[..^seq.Length]; + } + } + return text; + } + + private static async Task WriteEventAsync( + HttpContext ctx, string eventName, T payload, + System.Text.Json.Serialization.Metadata.JsonTypeInfo typeInfo, CancellationToken ct) + { + await ctx.Response.WriteAsync($"event: {eventName}\n", ct); + await ctx.Response.WriteAsync("data: ", ct); + await JsonSerializer.SerializeAsync(ctx.Response.Body, payload, typeInfo, ct); + await ctx.Response.WriteAsync("\n\n", ct); + await ctx.Response.Body.FlushAsync(ct); + } + + private static async Task WriteErrorAsync( + HttpContext ctx, int statusCode, string errorType, string message) + { + ctx.Response.StatusCode = statusCode; + await ctx.Response.WriteAsJsonAsync( + new AnthropicErrorResponse { Error = new AnthropicErrorBody { Type = errorType, Message = message } }, + ServerJsonContext.Default.AnthropicErrorResponse, + contentType: null, + ctx.RequestAborted); + } +} diff --git a/src/DotLLM.Server/Models/AnthropicMessagesModels.cs b/src/DotLLM.Server/Models/AnthropicMessagesModels.cs new file mode 100644 index 00000000..d5c21af8 --- /dev/null +++ b/src/DotLLM.Server/Models/AnthropicMessagesModels.cs @@ -0,0 +1,301 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace DotLLM.Server.Models; + +// --------------------------------------------------------------------------- +// Anthropic Messages API DTOs (POST /v1/messages, /v1/messages/count_tokens). +// +// These mirror the public Anthropic Messages API request/response shapes so +// that clients written for the `anthropic` SDKs can talk to dotLLM unchanged. +// The translation onto dotLLM engine types lives in AnthropicConverter; these +// records are pure serialization contracts. +// +// Reference: https://docs.anthropic.com/en/api/messages +// --------------------------------------------------------------------------- + +/// +/// Anthropic-compatible POST /v1/messages request. Also used (with +/// max_tokens/stream ignored) for POST /v1/messages/count_tokens. +/// +public sealed record AnthropicMessagesRequest +{ + [JsonPropertyName("model")] + public string? Model { get; init; } + + [JsonPropertyName("messages")] + public required AnthropicMessageDto[] Messages { get; init; } + + /// + /// Top-level system prompt. Either a string or an array of text content + /// blocks ({"type":"text","text":"..."}). Parsed by the converter. + /// + [JsonPropertyName("system")] + public JsonElement? System { get; init; } + + /// Maximum tokens to generate. Required by the Anthropic spec. + [JsonPropertyName("max_tokens")] + public int? MaxTokens { get; init; } + + [JsonPropertyName("stream")] + public bool Stream { get; init; } + + [JsonPropertyName("stop_sequences")] + public string[]? StopSequences { get; init; } + + [JsonPropertyName("temperature")] + public float? Temperature { get; init; } + + [JsonPropertyName("top_p")] + public float? TopP { get; init; } + + [JsonPropertyName("top_k")] + public int? TopK { get; init; } + + [JsonPropertyName("tools")] + public AnthropicToolDto[]? Tools { get; init; } + + /// + /// Tool choice: {"type":"auto"|"any"|"none"} or + /// {"type":"tool","name":"..."}. Parsed by the converter. + /// + [JsonPropertyName("tool_choice")] + public JsonElement? ToolChoice { get; init; } + + /// Opaque request metadata (e.g. user_id). Accepted and ignored. + [JsonPropertyName("metadata")] + public JsonElement? Metadata { get; init; } +} + +/// +/// A single Anthropic message. role is "user" or "assistant"; +/// content is either a string or an array of content blocks. +/// +public sealed record AnthropicMessageDto +{ + [JsonPropertyName("role")] + public required string Role { get; init; } + + [JsonPropertyName("content")] + public JsonElement Content { get; init; } +} + +/// Anthropic tool definition. input_schema is a JSON Schema object. +public sealed record AnthropicToolDto +{ + [JsonPropertyName("name")] + public required string Name { get; init; } + + [JsonPropertyName("description")] + public string? Description { get; init; } + + [JsonPropertyName("input_schema")] + public JsonElement? InputSchema { get; init; } +} + +// --- Response --------------------------------------------------------------- + +/// Anthropic-compatible non-streaming message response. +public sealed record AnthropicMessageResponse +{ + [JsonPropertyName("id")] + public required string Id { get; init; } + + [JsonPropertyName("type")] + public string Type { get; init; } = "message"; + + [JsonPropertyName("role")] + public string Role { get; init; } = "assistant"; + + [JsonPropertyName("content")] + public required AnthropicContentBlockDto[] Content { get; init; } + + [JsonPropertyName("model")] + public required string Model { get; init; } + + // stop_reason / stop_sequence are always emitted (even when null) to match + // the Anthropic wire format, overriding the context's WhenWritingNull default. + [JsonPropertyName("stop_reason")] + [JsonIgnore(Condition = JsonIgnoreCondition.Never)] + public string? StopReason { get; init; } + + [JsonPropertyName("stop_sequence")] + [JsonIgnore(Condition = JsonIgnoreCondition.Never)] + public string? StopSequence { get; init; } + + [JsonPropertyName("usage")] + public required AnthropicUsageDto Usage { get; init; } +} + +/// +/// A response content block. type is "text" or "tool_use"; +/// only the fields relevant to the type are populated (others omitted). +/// +public sealed record AnthropicContentBlockDto +{ + [JsonPropertyName("type")] + public required string Type { get; init; } + + [JsonPropertyName("text")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Text { get; init; } + + [JsonPropertyName("id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Id { get; init; } + + [JsonPropertyName("name")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Name { get; init; } + + [JsonPropertyName("input")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public JsonElement? Input { get; init; } +} + +/// Anthropic token usage (input_tokens/output_tokens). +public sealed record AnthropicUsageDto +{ + [JsonPropertyName("input_tokens")] + public int InputTokens { get; init; } + + [JsonPropertyName("output_tokens")] + public int OutputTokens { get; init; } +} + +/// POST /v1/messages/count_tokens response. +public sealed record AnthropicCountTokensResponse +{ + [JsonPropertyName("input_tokens")] + public int InputTokens { get; init; } +} + +// --- Error envelope --------------------------------------------------------- + +/// Anthropic error envelope: {"type":"error","error":{...}}. +public sealed record AnthropicErrorResponse +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "error"; + + [JsonPropertyName("error")] + public required AnthropicErrorBody Error { get; init; } +} + +/// Inner body of an Anthropic error envelope. +public sealed record AnthropicErrorBody +{ + /// Error category, e.g. invalid_request_error, api_error. + [JsonPropertyName("type")] + public required string Type { get; init; } + + [JsonPropertyName("message")] + public required string Message { get; init; } +} + +// --- Streaming events ------------------------------------------------------- +// Each is emitted as a named SSE event: `event: \ndata: \n\n`. + +/// message_start streaming event. +public sealed record AnthropicMessageStartEvent +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "message_start"; + + [JsonPropertyName("message")] + public required AnthropicMessageResponse Message { get; init; } +} + +/// content_block_start streaming event. +public sealed record AnthropicContentBlockStartEvent +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "content_block_start"; + + [JsonPropertyName("index")] + public int Index { get; init; } + + [JsonPropertyName("content_block")] + public required AnthropicContentBlockDto ContentBlock { get; init; } +} + +/// content_block_delta streaming event. +public sealed record AnthropicContentBlockDeltaEvent +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "content_block_delta"; + + [JsonPropertyName("index")] + public int Index { get; init; } + + [JsonPropertyName("delta")] + public required AnthropicStreamDeltaDto Delta { get; init; } +} + +/// +/// Incremental delta inside a content_block_delta event: +/// text_delta (carries text) or input_json_delta +/// (carries partial_json). +/// +public sealed record AnthropicStreamDeltaDto +{ + [JsonPropertyName("type")] + public required string Type { get; init; } + + [JsonPropertyName("text")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Text { get; init; } + + [JsonPropertyName("partial_json")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? PartialJson { get; init; } +} + +/// content_block_stop streaming event. +public sealed record AnthropicContentBlockStopEvent +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "content_block_stop"; + + [JsonPropertyName("index")] + public int Index { get; init; } +} + +/// message_delta streaming event (final stop reason + usage). +public sealed record AnthropicMessageDeltaEvent +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "message_delta"; + + [JsonPropertyName("delta")] + public required AnthropicMessageDeltaBody Delta { get; init; } + + [JsonPropertyName("usage")] + public required AnthropicUsageDto Usage { get; init; } +} + +/// Delta body of a message_delta event. +public sealed record AnthropicMessageDeltaBody +{ + [JsonPropertyName("stop_reason")] + [JsonIgnore(Condition = JsonIgnoreCondition.Never)] + public string? StopReason { get; init; } + + [JsonPropertyName("stop_sequence")] + [JsonIgnore(Condition = JsonIgnoreCondition.Never)] + public string? StopSequence { get; init; } +} + +/// message_stop streaming event. +public sealed record AnthropicMessageStopEvent +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "message_stop"; +} + +/// ping streaming keep-alive event. +public sealed record AnthropicPingEvent +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "ping"; +} diff --git a/src/DotLLM.Server/ServerJsonContext.cs b/src/DotLLM.Server/ServerJsonContext.cs index 885eac75..71c45dd4 100644 --- a/src/DotLLM.Server/ServerJsonContext.cs +++ b/src/DotLLM.Server/ServerJsonContext.cs @@ -28,6 +28,17 @@ namespace DotLLM.Server; [JsonSerializable(typeof(ModelInspectResponse))] [JsonSerializable(typeof(ErrorResponse))] [JsonSerializable(typeof(StatusResponse))] +[JsonSerializable(typeof(AnthropicMessagesRequest))] +[JsonSerializable(typeof(AnthropicMessageResponse))] +[JsonSerializable(typeof(AnthropicCountTokensResponse))] +[JsonSerializable(typeof(AnthropicErrorResponse))] +[JsonSerializable(typeof(AnthropicMessageStartEvent))] +[JsonSerializable(typeof(AnthropicContentBlockStartEvent))] +[JsonSerializable(typeof(AnthropicContentBlockDeltaEvent))] +[JsonSerializable(typeof(AnthropicContentBlockStopEvent))] +[JsonSerializable(typeof(AnthropicMessageDeltaEvent))] +[JsonSerializable(typeof(AnthropicMessageStopEvent))] +[JsonSerializable(typeof(AnthropicPingEvent))] [JsonSourceGenerationOptions( DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, PropertyNamingPolicy = JsonKnownNamingPolicy.SnakeCaseLower)] diff --git a/tests/DotLLM.Tests.Unit/Server/AnthropicConverterTests.cs b/tests/DotLLM.Tests.Unit/Server/AnthropicConverterTests.cs new file mode 100644 index 00000000..bf229af8 --- /dev/null +++ b/tests/DotLLM.Tests.Unit/Server/AnthropicConverterTests.cs @@ -0,0 +1,294 @@ +using System.Text.Json; +using DotLLM.Engine; +using DotLLM.Server; +using DotLLM.Server.Endpoints; +using DotLLM.Server.Models; +using DotLLM.Tokenizers; +using Xunit; + +namespace DotLLM.Tests.Unit.Server; + +/// +/// Unit tests for the Anthropic Messages API translation layer +/// ( and validation). +/// These exercise pure request/response reshaping — no model load required. +/// +public sealed class AnthropicConverterTests +{ + private static AnthropicMessagesRequest Parse(string json) => + JsonSerializer.Deserialize(json, ServerJsonContext.Default.AnthropicMessagesRequest)!; + + // --- Message flattening ------------------------------------------------- + + [Fact] + public void ToMessages_StringContent_MapsOneToOne() + { + var req = Parse(""" + {"model":"m","max_tokens":16,"messages":[{"role":"user","content":"Hello"}]} + """); + + var messages = AnthropicConverter.ToMessages(req); + + Assert.Single(messages); + Assert.Equal("user", messages[0].Role); + Assert.Equal("Hello", messages[0].Content); + } + + [Fact] + public void ToMessages_SystemString_BecomesLeadingSystemMessage() + { + var req = Parse(""" + {"model":"m","max_tokens":16,"system":"Be terse.","messages":[{"role":"user","content":"hi"}]} + """); + + var messages = AnthropicConverter.ToMessages(req); + + Assert.Equal(2, messages.Length); + Assert.Equal("system", messages[0].Role); + Assert.Equal("Be terse.", messages[0].Content); + Assert.Equal("user", messages[1].Role); + } + + [Fact] + public void ToMessages_SystemBlockArray_IsConcatenated() + { + var req = Parse(""" + {"model":"m","max_tokens":16, + "system":[{"type":"text","text":"Line 1"},{"type":"text","text":"Line 2"}], + "messages":[{"role":"user","content":"hi"}]} + """); + + var messages = AnthropicConverter.ToMessages(req); + + Assert.Equal("system", messages[0].Role); + Assert.Equal("Line 1\nLine 2", messages[0].Content); + } + + [Fact] + public void ToMessages_TextBlockArray_IsConcatenated() + { + var req = Parse(""" + {"model":"m","max_tokens":16,"messages":[ + {"role":"user","content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]}]} + """); + + var messages = AnthropicConverter.ToMessages(req); + + Assert.Single(messages); + Assert.Equal("a\nb", messages[0].Content); + } + + [Fact] + public void ToMessages_AssistantToolUseBlock_BecomesToolCall() + { + var req = Parse(""" + {"model":"m","max_tokens":16,"messages":[ + {"role":"assistant","content":[ + {"type":"tool_use","id":"toolu_1","name":"get_weather","input":{"city":"Paris"}}]}]} + """); + + var messages = AnthropicConverter.ToMessages(req); + + Assert.Single(messages); + Assert.Equal("assistant", messages[0].Role); + Assert.NotNull(messages[0].ToolCalls); + var call = messages[0].ToolCalls![0]; + Assert.Equal("toolu_1", call.Id); + Assert.Equal("get_weather", call.FunctionName); + Assert.Contains("Paris", call.Arguments); + } + + [Fact] + public void ToMessages_ToolResultBlock_BecomesToolRoleMessage() + { + var req = Parse(""" + {"model":"m","max_tokens":16,"messages":[ + {"role":"user","content":[ + {"type":"tool_result","tool_use_id":"toolu_1","content":"22C and sunny"}]}]} + """); + + var messages = AnthropicConverter.ToMessages(req); + + Assert.Single(messages); + Assert.Equal("tool", messages[0].Role); + Assert.Equal("toolu_1", messages[0].ToolCallId); + Assert.Equal("22C and sunny", messages[0].Content); + } + + [Fact] + public void ToMessages_ToolResultBlockArrayContent_ConcatenatesText() + { + var req = Parse(""" + {"model":"m","max_tokens":16,"messages":[ + {"role":"user","content":[ + {"type":"tool_result","tool_use_id":"toolu_2","content":[{"type":"text","text":"ok"}]}]}]} + """); + + var messages = AnthropicConverter.ToMessages(req); + + Assert.Equal("tool", messages[0].Role); + Assert.Equal("ok", messages[0].Content); + } + + // --- Tool choice -------------------------------------------------------- + + [Theory] + [InlineData("""{"type":"auto"}""", typeof(DotLLM.Core.Configuration.ToolChoice.Auto))] + [InlineData("""{"type":"any"}""", typeof(DotLLM.Core.Configuration.ToolChoice.Required))] + [InlineData("""{"type":"none"}""", typeof(DotLLM.Core.Configuration.ToolChoice.None))] + public void ParseToolChoice_MapsTypes(string json, Type expected) + { + using var doc = JsonDocument.Parse(json); + var choice = AnthropicConverter.ParseToolChoice(doc.RootElement); + Assert.IsType(expected, choice); + } + + [Fact] + public void ParseToolChoice_Tool_MapsToFunction() + { + using var doc = JsonDocument.Parse("""{"type":"tool","name":"get_weather"}"""); + var choice = AnthropicConverter.ParseToolChoice(doc.RootElement); + var fn = Assert.IsType(choice); + Assert.Equal("get_weather", fn.Name); + } + + [Fact] + public void ParseToolChoice_Null_DefaultsToAuto() + { + Assert.IsType(AnthropicConverter.ParseToolChoice(null)); + } + + // --- Stop reason mapping ------------------------------------------------ + + [Theory] + [InlineData(FinishReason.Stop, false, "end_turn")] + [InlineData(FinishReason.Stop, true, "stop_sequence")] + [InlineData(FinishReason.Length, false, "max_tokens")] + [InlineData(FinishReason.ToolCalls, false, "tool_use")] + public void ToStopReason_MapsFinishReason(FinishReason reason, bool matched, string expected) + { + Assert.Equal(expected, AnthropicConverter.ToStopReason(reason, matched)); + } + + // --- Tool use blocks ---------------------------------------------------- + + [Fact] + public void ToToolUseBlocks_ParsesArgumentsIntoJsonObject() + { + var blocks = AnthropicConverter.ToToolUseBlocks( + [new ToolCall("toolu_x", "get_weather", """{"city":"Paris"}""")]); + + var block = Assert.Single(blocks); + Assert.Equal("tool_use", block.Type); + Assert.Equal("toolu_x", block.Id); + Assert.Equal("get_weather", block.Name); + Assert.NotNull(block.Input); + Assert.Equal("Paris", block.Input!.Value.GetProperty("city").GetString()); + } + + [Fact] + public void ToToolUseBlocks_MissingId_GeneratesAnthropicId() + { + var blocks = AnthropicConverter.ToToolUseBlocks([new ToolCall("", "f", "{}")]); + Assert.StartsWith("toolu_", blocks[0].Id); + } + + [Fact] + public void ParseInput_InvalidJson_ReturnsEmptyObject() + { + var el = AnthropicConverter.ParseInput("not json"); + Assert.Equal(JsonValueKind.Object, el.ValueKind); + Assert.False(el.EnumerateObject().MoveNext()); + } + + // --- Request validation ------------------------------------------------- + + [Fact] + public void ValidateRequest_EmptyMessages_Fails() + { + var req = new AnthropicMessagesRequest { Messages = [], MaxTokens = 16 }; + Assert.NotNull(MessagesEndpoint.ValidateRequest(req, requireMaxTokens: true)); + } + + [Fact] + public void ValidateRequest_MissingMaxTokens_FailsWhenRequired() + { + var req = Parse("""{"model":"m","messages":[{"role":"user","content":"hi"}]}"""); + Assert.NotNull(MessagesEndpoint.ValidateRequest(req, requireMaxTokens: true)); + // count_tokens does not require max_tokens. + Assert.Null(MessagesEndpoint.ValidateRequest(req, requireMaxTokens: false)); + } + + [Fact] + public void ValidateRequest_NonPositiveMaxTokens_Fails() + { + var req = new AnthropicMessagesRequest + { + Messages = [new AnthropicMessageDto { Role = "user", Content = default }], + MaxTokens = 0, + }; + Assert.NotNull(MessagesEndpoint.ValidateRequest(req, requireMaxTokens: true)); + } + + [Fact] + public void ValidateRequest_Valid_ReturnsNull() + { + var req = Parse(""" + {"model":"m","max_tokens":16,"messages":[{"role":"user","content":"hi"}]} + """); + Assert.Null(MessagesEndpoint.ValidateRequest(req, requireMaxTokens: true)); + } + + // --- Response serialization shape -------------------------------------- + + [Fact] + public void MessageResponse_SerializesAnthropicShape() + { + var response = new AnthropicMessageResponse + { + Id = "msg_1", + Model = "m", + Content = [new AnthropicContentBlockDto { Type = "text", Text = "Hi" }], + StopReason = "end_turn", + StopSequence = null, + Usage = new AnthropicUsageDto { InputTokens = 5, OutputTokens = 2 }, + }; + + string json = JsonSerializer.Serialize(response, ServerJsonContext.Default.AnthropicMessageResponse); + + using var doc = JsonDocument.Parse(json); + var root = doc.RootElement; + Assert.Equal("message", root.GetProperty("type").GetString()); + Assert.Equal("assistant", root.GetProperty("role").GetString()); + Assert.Equal("end_turn", root.GetProperty("stop_reason").GetString()); + // stop_sequence must be present even when null (Anthropic wire format). + Assert.Equal(JsonValueKind.Null, root.GetProperty("stop_sequence").ValueKind); + Assert.Equal(5, root.GetProperty("usage").GetProperty("input_tokens").GetInt32()); + Assert.Equal("text", root.GetProperty("content")[0].GetProperty("type").GetString()); + Assert.Equal("Hi", root.GetProperty("content")[0].GetProperty("text").GetString()); + } + + [Fact] + public void CountTokensResponse_SerializesInputTokens() + { + string json = JsonSerializer.Serialize( + new AnthropicCountTokensResponse { InputTokens = 42 }, + ServerJsonContext.Default.AnthropicCountTokensResponse); + using var doc = JsonDocument.Parse(json); + Assert.Equal(42, doc.RootElement.GetProperty("input_tokens").GetInt32()); + } + + [Fact] + public void ErrorResponse_SerializesAnthropicEnvelope() + { + string json = JsonSerializer.Serialize( + new AnthropicErrorResponse + { + Error = new AnthropicErrorBody { Type = "invalid_request_error", Message = "bad" }, + }, + ServerJsonContext.Default.AnthropicErrorResponse); + using var doc = JsonDocument.Parse(json); + Assert.Equal("error", doc.RootElement.GetProperty("type").GetString()); + Assert.Equal("invalid_request_error", doc.RootElement.GetProperty("error").GetProperty("type").GetString()); + } +} From 85b64dea2f4fb262cc87c8f6dafca7c1d66f31f6 Mon Sep 17 00:00:00 2001 From: James Burton Date: Fri, 31 Jul 2026 11:11:34 +0100 Subject: [PATCH 2/2] server: address Copilot review on the Anthropic Messages API (#325) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on the /v1/messages surface: - ParseInput: Anthropic requires tool_use.input to be an object, so a non-object JSON root (bare string/array/scalar from a model that ignored the schema) now collapses to {} instead of being emitted as an invalid wire shape. - ParseInput/EmptyObject: share one cloned empty-object JsonElement instead of reparsing "{}" on every empty or invalid tool input. - ValidateRequest: reject messages[].role other than user/assistant, and messages[].content that is neither a string nor an array of blocks. The converter passes role straight to the chat template, so an unchecked role let a caller inject a system turn mid-conversation; an unchecked content kind silently flattened to an empty message. - Streaming: drop the `Connection: keep-alive` response header — it is a connection-specific header that is illegal over HTTP/2 and HTTP/3, and SSE only needs content-type + no-cache. - Extract the SSE emission into internal WriteMessageStreamAsync with the token source and the model-serialisation gate injected, so the event sequence is testable without loading a model. Behaviour is unchanged: the gate still wraps only the generation loop, so message_start is emitted before queueing. - docs/ANTHROPIC_API.md: drop the lora_adapter example — per-request adapter selection is not implemented on the server (the OpenAI surface does not honour it either); document the role restriction instead. Tests: 49 (was 26) — endpoint-level SSE tests asserting event names, ordering and payload shapes for text-only, max_tokens, stop_sequence and tool_use streams, plus ParseInput non-object and role/content validation regressions. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 2 +- docs/ANTHROPIC_API.md | 9 +- src/DotLLM.Server/AnthropicConverter.cs | 24 ++- .../Endpoints/MessagesEndpoint.cs | 65 +++++- .../Server/AnthropicConverterTests.cs | 70 +++++++ .../Server/AnthropicStreamingTests.cs | 192 ++++++++++++++++++ 6 files changed, 346 insertions(+), 16 deletions(-) create mode 100644 tests/DotLLM.Tests.Unit/Server/AnthropicStreamingTests.cs diff --git a/README.md b/README.md index d27feb7b..f33cbe5e 100644 --- a/README.md +++ b/README.md @@ -673,7 +673,7 @@ Both modes transparently reuse the embedded chat UI assets if `serveUi: true`. T ## News -- **2026-06** — **Anthropic-compatible Messages API** — `/v1/messages` (non-streaming + event-based SSE: `message_start`, `content_block_start`/`_delta`/`_stop`, `message_delta`, `message_stop`) and `/v1/messages/count_tokens`, served alongside the OpenAI surface. Top-level `system` (string or text-block array), string-or-block message `content`, `tool_use`/`tool_result` content blocks, `tool_choice` (`auto`/`any`/`none`/`tool`), and `FinishReason → stop_reason` mapping (`end_turn`/`max_tokens`/`stop_sequence`/`tool_use`). Reuses the shared engine, chat-template, sampler and tool-calling pipeline — only the wire format differs (`MessagesEndpoint` + `AnthropicConverter`). 26 unit tests; see `docs/ANTHROPIC_API.md` ([#325](https://github.com/kkokosa/dotLLM/issues/325)) +- **2026-06** — **Anthropic-compatible Messages API** — `/v1/messages` (non-streaming + event-based SSE: `message_start`, `content_block_start`/`_delta`/`_stop`, `message_delta`, `message_stop`) and `/v1/messages/count_tokens`, served alongside the OpenAI surface. Top-level `system` (string or text-block array), string-or-block message `content`, `tool_use`/`tool_result` content blocks, `tool_choice` (`auto`/`any`/`none`/`tool`), and `FinishReason → stop_reason` mapping (`end_turn`/`max_tokens`/`stop_sequence`/`tool_use`). Reuses the shared engine, chat-template, sampler and tool-calling pipeline — only the wire format differs (`MessagesEndpoint` + `AnthropicConverter`). 49 unit tests (including endpoint-level SSE event-sequence coverage); see `docs/ANTHROPIC_API.md` ([#325](https://github.com/kkokosa/dotLLM/issues/325)) - **2026-04** — **First public release (v0.1.0-preview.1)** — dotLLM goes public. [NuGet packages](#nuget-packages) for all 10 libraries + `DotLLM.Cli` as a global `dotnet tool`. Self-contained single-file downloads for Windows / Linux / macOS (Apple Silicon) and experimental Native AOT builds for Linux / Windows attached to every [GitHub Release](https://github.com/kkokosa/dotLLM/releases). Companion website at [dotllm.dev](https://dotllm.dev/) ([#119](https://github.com/kkokosa/dotLLM/issues/119)) - **2026-04** — **Wave 7**: CPU performance cleanup pass — `TopKSampler` replaces full `Array.Sort` with a hand-rolled size-K min-heap (`O(N log K)`, stack-resident scratch); `JsonSchemaConstraint` adds first-char bucketing to skip the ~160 MB of struct clones per mask build when the tracker rejects most leading characters, plus LRU eviction instead of the previous full-flush cache overflow; `Dequantize.Q5_0` gains an AVX2 path matching Q8_0's throughput (reuses `MatMulQ5_0.ExtractQ5HighBits` / `vpshufb` bit-extraction); `BpeTokenizer` pre-splits special tokens via the existing `Trie.TryMatchLongest` instead of the O(n × m) linear scan; `ComputeThreadPool` now pins the caller (inference) thread to the first candidate P-core on first `Dispatch`, eliminating the hybrid-CPU stall where pinned P-core workers idled at the barrier waiting for an E-core caller. New BenchmarkDotNet suites for TopK sampling, schema mask build, and special-token encode ([#109](https://github.com/kkokosa/dotLLM/issues/109)) - **2026-04** — **Phase 7 begins**: Logprobs — OpenAI-compatible `logprobs: true` + `top_logprobs: N` (0-20) on `/v1/chat/completions` and `/v1/completions`. Per-token log-softmax captured before sampling, returned in both streaming SSE chunks and non-streaming responses. Chat UI gains opt-in logprobs visualization: color-coded token confidence (green/lime/yellow/orange/red), hover tooltips with top-K alternatives and probabilities, diagnostic cues for low confidence, ambiguity, and sampling effect. `DotLLM.Sample.Logprobs` console sample with ANSI-colored output ([#101](https://github.com/kkokosa/dotLLM/issues/101)) diff --git a/docs/ANTHROPIC_API.md b/docs/ANTHROPIC_API.md index f1d97045..7c7436bd 100644 --- a/docs/ANTHROPIC_API.md +++ b/docs/ANTHROPIC_API.md @@ -39,8 +39,7 @@ non-streaming (JSON) and streaming (named SSE events). "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}}} ], "tool_choice": {"type": "auto"}, - "stream": false, - "lora_adapter": "customer-support" + "stream": false } ``` @@ -51,8 +50,12 @@ non-streaming (JSON) and streaming (named SSE events). (`text`, `tool_use`, `tool_result`). - `tool_choice`: `{"type":"auto"}`, `{"type":"any"}` (→ required), `{"type":"none"}`, or `{"type":"tool","name":"..."}`. -- `lora_adapter` is a dotLLM extension (parity with the OpenAI surface). +- `messages[].role` must be `user` or `assistant`; any other role → `400`. + (The top-level `system` field is the only way to set a system prompt.) - `image` content blocks are not yet supported (no multimodal pipeline). +- `lora_adapter` is **not** honoured by this endpoint — per-request adapter + selection is not implemented on the server yet (the OpenAI surface does not + honour it either). Unknown fields are ignored, not rejected. **Response** (non-streaming): ```json diff --git a/src/DotLLM.Server/AnthropicConverter.cs b/src/DotLLM.Server/AnthropicConverter.cs index d71c3cb5..9135471d 100644 --- a/src/DotLLM.Server/AnthropicConverter.cs +++ b/src/DotLLM.Server/AnthropicConverter.cs @@ -255,23 +255,37 @@ public static AnthropicContentBlockDto[] ToToolUseBlocks(ToolCall[] toolCalls) return blocks; } - /// Parses a tool-call argument JSON string into a JSON object element. + /// + /// Parses a tool-call argument JSON string into a JSON object element. + /// + /// + /// Anthropic requires tool_use.input to be an object, so a non-object root + /// (a bare string, array or number emitted by a model that ignored the schema) + /// collapses to {} rather than being passed through as an invalid wire shape + /// that clients would reject or mis-handle. + /// public static JsonElement ParseInput(string? arguments) { if (string.IsNullOrWhiteSpace(arguments)) - return EmptyObject(); + return EmptyObject; try { using var doc = JsonDocument.Parse(arguments); - return doc.RootElement.Clone(); + return doc.RootElement.ValueKind == JsonValueKind.Object + ? doc.RootElement.Clone() + : EmptyObject; } catch (JsonException) { - return EmptyObject(); + return EmptyObject; } } - private static JsonElement EmptyObject() + // A cloned JsonElement is detached and immutable, so one instance can be shared + // across every empty/invalid tool input instead of reparsing "{}" per tool call. + private static readonly JsonElement EmptyObject = ParseEmptyObject(); + + private static JsonElement ParseEmptyObject() { using var doc = JsonDocument.Parse("{}"); return doc.RootElement.Clone(); diff --git a/src/DotLLM.Server/Endpoints/MessagesEndpoint.cs b/src/DotLLM.Server/Endpoints/MessagesEndpoint.cs index 94541a5f..36469047 100644 --- a/src/DotLLM.Server/Endpoints/MessagesEndpoint.cs +++ b/src/DotLLM.Server/Endpoints/MessagesEndpoint.cs @@ -164,10 +164,43 @@ private static async Task HandleStreamingAsync( ToolDefinition[]? tools, int promptTokenCount, CancellationToken ct) + => await WriteMessageStreamAsync( + httpContext, + innerCt => generator.GenerateStreamingTokensAsync(prompt, options, innerCt), + state.ExecuteAsync, + tools is { Length: > 0 } ? state.ToolCallParser : null, + request.StopSequences, + messageId, modelId, promptTokenCount, ct); + + /// + /// Emits the Anthropic SSE event sequence for one streaming request: + /// message_start, content_block_start + ping, a + /// content_block_delta per generated token, content_block_stop, + /// an optional start/delta/stop trio per detected tool_use block, then + /// message_delta and message_stop. + /// + /// + /// The token source and the model-serialisation gate are injected rather than read + /// from so the emitted event sequence can be asserted in + /// unit tests without loading a model. wraps only the + /// generation loop, so message_start still reaches the client before the + /// request queues behind the model lock. + /// + internal static async Task WriteMessageStreamAsync( + HttpContext httpContext, + Func> tokenSource, + Func, CancellationToken, Task> execute, + IToolCallParser? toolCallParser, + string[]? requestStopSequences, + string messageId, + string modelId, + int promptTokenCount, + CancellationToken ct) { httpContext.Response.ContentType = "text/event-stream"; + // No `Connection: keep-alive` — it is a connection-specific header that is + // illegal over HTTP/2 and HTTP/3, and content-type + no-cache is all SSE needs. httpContext.Response.Headers.CacheControl = "no-cache"; - httpContext.Response.Headers.Connection = "keep-alive"; // message_start — input_tokens known up front from the prompt. var startMessage = new AnthropicMessageResponse @@ -198,9 +231,9 @@ await WriteEventAsync(httpContext, "content_block_start", FinishReason finishReason = FinishReason.Length; int completionTokens = 0; - await state.ExecuteAsync(async () => + await execute(async () => { - await foreach (var token in generator.GenerateStreamingTokensAsync(prompt, options, ct)) + await foreach (var token in tokenSource(ct)) { if (token.Text.Length > 0) { @@ -228,9 +261,9 @@ await WriteEventAsync(httpContext, "content_block_stop", // Post-generation tool-call detection (mirrors the OpenAI streaming endpoint). string text = sb.ToString(); ToolCall[]? toolCalls = null; - if (state.ToolCallParser is not null && tools is { Length: > 0 }) + if (toolCallParser is not null) { - toolCalls = state.ToolCallParser.TryParse(text); + toolCalls = toolCallParser.TryParse(text); if (toolCalls is { Length: > 0 }) finishReason = FinishReason.ToolCalls; } @@ -275,11 +308,11 @@ await WriteEventAsync(httpContext, "content_block_stop", bool matchedStopSequence = false; if (finishReason == FinishReason.Stop) - matchedStopSequence = MatchStopSequence(text, request.StopSequences) is not null; + matchedStopSequence = MatchStopSequence(text, requestStopSequences) is not null; string stopReason = AnthropicConverter.ToStopReason(finishReason, matchedStopSequence); string? stopSequence = stopReason == "stop_sequence" - ? MatchStopSequence(text, request.StopSequences) + ? MatchStopSequence(text, requestStopSequences) : null; await WriteEventAsync(httpContext, "message_delta", @@ -344,6 +377,24 @@ await JsonSerializer.SerializeAsync(httpContext.Response.Body, return "max_tokens: must be a positive integer"; } + // Roles and content kinds are checked here rather than left to the converter: + // ToMessages passes `role` straight through to the chat template, so an unchecked + // `system` (or arbitrary) role would let a caller inject a system turn mid- + // conversation, and an unchecked content kind would silently flatten to an empty + // message instead of surfacing the client's mistake as a 400. + for (int i = 0; i < request.Messages.Length; i++) + { + var msg = request.Messages[i]; + if (msg is null) + return $"messages[{i}]: must be an object"; + + if (msg.Role is not ("user" or "assistant")) + return $"messages[{i}].role: must be one of \"user\", \"assistant\""; + + if (msg.Content.ValueKind is not (JsonValueKind.String or JsonValueKind.Array)) + return $"messages[{i}].content: must be a string or an array of content blocks"; + } + return null; } diff --git a/tests/DotLLM.Tests.Unit/Server/AnthropicConverterTests.cs b/tests/DotLLM.Tests.Unit/Server/AnthropicConverterTests.cs index bf229af8..1c173166 100644 --- a/tests/DotLLM.Tests.Unit/Server/AnthropicConverterTests.cs +++ b/tests/DotLLM.Tests.Unit/Server/AnthropicConverterTests.cs @@ -201,6 +201,30 @@ public void ParseInput_InvalidJson_ReturnsEmptyObject() Assert.False(el.EnumerateObject().MoveNext()); } + [Theory] + [InlineData("\"hi\"")] + [InlineData("[]")] + [InlineData("[1,2]")] + [InlineData("42")] + [InlineData("null")] + [InlineData("true")] + public void ParseInput_NonObjectRoot_ReturnsEmptyObject(string arguments) + { + // Anthropic requires tool_use.input to be an object — a model that emits a bare + // scalar or array must not produce an invalid wire shape. + var el = AnthropicConverter.ParseInput(arguments); + Assert.Equal(JsonValueKind.Object, el.ValueKind); + Assert.False(el.EnumerateObject().MoveNext()); + } + + [Fact] + public void ParseInput_ObjectRoot_PassesThrough() + { + var el = AnthropicConverter.ParseInput("""{"city":"Paris"}"""); + Assert.Equal(JsonValueKind.Object, el.ValueKind); + Assert.Equal("Paris", el.GetProperty("city").GetString()); + } + // --- Request validation ------------------------------------------------- [Fact] @@ -230,6 +254,52 @@ public void ValidateRequest_NonPositiveMaxTokens_Fails() Assert.NotNull(MessagesEndpoint.ValidateRequest(req, requireMaxTokens: true)); } + [Theory] + [InlineData("system")] + [InlineData("tool")] + [InlineData("developer")] + [InlineData("")] + public void ValidateRequest_UnsupportedRole_Fails(string role) + { + // Roles flow straight into the chat template; only user/assistant are addressable + // by a client (the system prompt is the top-level `system` field). + var req = Parse($$""" + {"model":"m","max_tokens":16,"messages":[{"role":"{{role}}","content":"hi"}]} + """); + Assert.NotNull(MessagesEndpoint.ValidateRequest(req, requireMaxTokens: true)); + } + + [Theory] + [InlineData("123")] + [InlineData("null")] + [InlineData("true")] + [InlineData("""{"type":"text"}""")] + public void ValidateRequest_NonStringNonArrayContent_Fails(string content) + { + var req = Parse($$""" + {"model":"m","max_tokens":16,"messages":[{"role":"user","content":{{content}}}]} + """); + Assert.NotNull(MessagesEndpoint.ValidateRequest(req, requireMaxTokens: true)); + } + + [Fact] + public void ValidateRequest_MissingContent_Fails() + { + var req = Parse("""{"model":"m","max_tokens":16,"messages":[{"role":"user"}]}"""); + Assert.NotNull(MessagesEndpoint.ValidateRequest(req, requireMaxTokens: true)); + } + + [Fact] + public void ValidateRequest_BlockArrayContent_ReturnsNull() + { + var req = Parse(""" + {"model":"m","max_tokens":16,"messages":[ + {"role":"user","content":[{"type":"text","text":"hi"}]}, + {"role":"assistant","content":"yes"}]} + """); + Assert.Null(MessagesEndpoint.ValidateRequest(req, requireMaxTokens: true)); + } + [Fact] public void ValidateRequest_Valid_ReturnsNull() { diff --git a/tests/DotLLM.Tests.Unit/Server/AnthropicStreamingTests.cs b/tests/DotLLM.Tests.Unit/Server/AnthropicStreamingTests.cs new file mode 100644 index 00000000..ffd83f0f --- /dev/null +++ b/tests/DotLLM.Tests.Unit/Server/AnthropicStreamingTests.cs @@ -0,0 +1,192 @@ +using System.Text; +using System.Text.Json; +using DotLLM.Core.Configuration; +using DotLLM.Engine; +using DotLLM.Server.Endpoints; +using DotLLM.Tokenizers; +using Microsoft.AspNetCore.Http; +using Xunit; + +namespace DotLLM.Tests.Unit.Server; + +/// +/// Endpoint-level tests for the Anthropic streaming SSE surface: they drive +/// with a scripted token stream +/// and assert the emitted event names, ordering and JSON payload shapes. No model is +/// loaded — the token source and the model gate are injected. +/// +public sealed class AnthropicStreamingTests +{ + /// One SSE frame: the event: name and its parsed data: payload. + private readonly record struct SseFrame(string Event, JsonElement Data); + + private static async IAsyncEnumerable Tokens( + params (string Text, FinishReason? Finish)[] script) + { + foreach (var (text, finish) in script) + { + await Task.Yield(); + yield return new GenerationToken(0, text, finish); + } + } + + /// Pass-through gate standing in for ServerState.ExecuteAsync. + private static Task NoGate(Func work, CancellationToken ct) => work(); + + private sealed class FixedToolCallParser(ToolCall[]? result) : IToolCallParser + { + public ToolCall[]? TryParse(string generatedText) => result; + public bool IsToolCallStart(string text) => false; + } + + private static async Task RunAsync( + IAsyncEnumerable tokens, + IToolCallParser? parser = null, + string[]? stopSequences = null) + { + var ctx = new DefaultHttpContext(); + var body = new MemoryStream(); + ctx.Response.Body = body; + + await MessagesEndpoint.WriteMessageStreamAsync( + ctx, _ => tokens, NoGate, parser, stopSequences, + messageId: "msg_test", modelId: "test-model", promptTokenCount: 7, + CancellationToken.None); + + Assert.Equal("text/event-stream", ctx.Response.ContentType); + // Connection-specific headers are illegal over HTTP/2 and must not be emitted. + Assert.False(ctx.Response.Headers.ContainsKey("Connection")); + + return ParseSse(Encoding.UTF8.GetString(body.ToArray())); + } + + private static SseFrame[] ParseSse(string raw) + { + var frames = new List(); + foreach (var block in raw.Split("\n\n", StringSplitOptions.RemoveEmptyEntries)) + { + var lines = block.Split('\n', StringSplitOptions.RemoveEmptyEntries); + string name = lines[0]["event: ".Length..]; + string data = lines[1]["data: ".Length..]; + frames.Add(new SseFrame(name, JsonDocument.Parse(data).RootElement.Clone())); + } + return [.. frames]; + } + + // --- Text-only stream --------------------------------------------------- + + [Fact] + public async Task Streaming_TextOnly_EmitsExpectedEventSequence() + { + var frames = await RunAsync(Tokens(("Hel", null), ("lo", FinishReason.Stop))); + + Assert.Equal( + ["message_start", "content_block_start", "ping", "content_block_delta", + "content_block_delta", "content_block_stop", "message_delta", "message_stop"], + frames.Select(f => f.Event)); + } + + [Fact] + public async Task Streaming_TextOnly_EmitsExpectedPayloadShapes() + { + var frames = await RunAsync(Tokens(("Hel", null), ("lo", FinishReason.Stop))); + + var start = frames[0].Data; + Assert.Equal("message_start", start.GetProperty("type").GetString()); + var startMsg = start.GetProperty("message"); + Assert.Equal("msg_test", startMsg.GetProperty("id").GetString()); + Assert.Equal("message", startMsg.GetProperty("type").GetString()); + Assert.Equal("assistant", startMsg.GetProperty("role").GetString()); + Assert.Equal("test-model", startMsg.GetProperty("model").GetString()); + Assert.Equal(JsonValueKind.Null, startMsg.GetProperty("stop_reason").ValueKind); + Assert.Equal(7, startMsg.GetProperty("usage").GetProperty("input_tokens").GetInt32()); + + var blockStart = frames[1].Data; + Assert.Equal(0, blockStart.GetProperty("index").GetInt32()); + Assert.Equal("text", blockStart.GetProperty("content_block").GetProperty("type").GetString()); + + var delta = frames[3].Data.GetProperty("delta"); + Assert.Equal("text_delta", delta.GetProperty("type").GetString()); + Assert.Equal("Hel", delta.GetProperty("text").GetString()); + Assert.Equal("lo", frames[4].Data.GetProperty("delta").GetProperty("text").GetString()); + + Assert.Equal(0, frames[5].Data.GetProperty("index").GetInt32()); + + var messageDelta = frames[6].Data; + Assert.Equal("end_turn", messageDelta.GetProperty("delta").GetProperty("stop_reason").GetString()); + Assert.Equal(JsonValueKind.Null, messageDelta.GetProperty("delta").GetProperty("stop_sequence").ValueKind); + Assert.Equal(2, messageDelta.GetProperty("usage").GetProperty("output_tokens").GetInt32()); + + Assert.Equal("message_stop", frames[7].Data.GetProperty("type").GetString()); + } + + [Fact] + public async Task Streaming_MaxTokens_ReportsMaxTokensStopReason() + { + var frames = await RunAsync(Tokens(("hi", FinishReason.Length))); + + var messageDelta = frames.Single(f => f.Event == "message_delta").Data; + Assert.Equal("max_tokens", messageDelta.GetProperty("delta").GetProperty("stop_reason").GetString()); + } + + [Fact] + public async Task Streaming_MatchingStopSequence_ReportsStopSequence() + { + var frames = await RunAsync( + Tokens(("all done", null), ("END", FinishReason.Stop)), + stopSequences: ["END"]); + + var delta = frames.Single(f => f.Event == "message_delta").Data.GetProperty("delta"); + Assert.Equal("stop_sequence", delta.GetProperty("stop_reason").GetString()); + Assert.Equal("END", delta.GetProperty("stop_sequence").GetString()); + } + + // --- tool_use stream ---------------------------------------------------- + + [Fact] + public async Task Streaming_ToolUse_EmitsToolBlockAfterTextBlock() + { + var parser = new FixedToolCallParser( + [new ToolCall("toolu_1", "get_weather", """{"city":"Paris"}""")]); + var frames = await RunAsync(Tokens(("calling", FinishReason.Stop)), parser); + + Assert.Equal( + ["message_start", "content_block_start", "ping", "content_block_delta", + "content_block_stop", "content_block_start", "content_block_delta", + "content_block_stop", "message_delta", "message_stop"], + frames.Select(f => f.Event)); + + // The tool_use block opens at index 1, after the text block at index 0. + var toolStart = frames[5].Data; + Assert.Equal(1, toolStart.GetProperty("index").GetInt32()); + var block = toolStart.GetProperty("content_block"); + Assert.Equal("tool_use", block.GetProperty("type").GetString()); + Assert.Equal("toolu_1", block.GetProperty("id").GetString()); + Assert.Equal("get_weather", block.GetProperty("name").GetString()); + // Anthropic opens a tool_use block with an empty input; arguments arrive as deltas. + Assert.Equal(JsonValueKind.Object, block.GetProperty("input").ValueKind); + Assert.Empty(block.GetProperty("input").EnumerateObject()); + + var toolDelta = frames[6].Data.GetProperty("delta"); + Assert.Equal("input_json_delta", toolDelta.GetProperty("type").GetString()); + Assert.Equal("""{"city":"Paris"}""", toolDelta.GetProperty("partial_json").GetString()); + + Assert.Equal(1, frames[7].Data.GetProperty("index").GetInt32()); + + var messageDelta = frames[8].Data; + Assert.Equal("tool_use", messageDelta.GetProperty("delta").GetProperty("stop_reason").GetString()); + } + + [Fact] + public async Task Streaming_ToolParserFindsNothing_EmitsTextOnlySequence() + { + var frames = await RunAsync(Tokens(("plain", FinishReason.Stop)), new FixedToolCallParser(null)); + + Assert.DoesNotContain(frames, f => + f.Event == "content_block_start" && + f.Data.GetProperty("content_block").GetProperty("type").GetString() == "tool_use"); + Assert.Equal("end_turn", + frames.Single(f => f.Event == "message_delta").Data.GetProperty("delta") + .GetProperty("stop_reason").GetString()); + } +}