From 14d6f0a922bce90a15e168e8eb38f6b64b5ee7f4 Mon Sep 17 00:00:00 2001 From: sixvolts Date: Mon, 7 Sep 2026 20:51:11 +0000 Subject: [PATCH 1/2] feat(pipeline): let the classifier drive the model's reasoning effort MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chat felt like it dragged on every turn, and the suspicion was that something was pushing the model to xhigh. It was not — the opposite. Familiar had no way to set effort at all. What was actually happening. The only thinking control on the wire was chat_template_kwargs.enable_thinking, a bool. The classifier's Off/Low/Medium/ High level resolved to a TOKEN BUDGET, and on the trusted chat path even that was discarded: pipeline.go deliberately grants the High budget (8000) every time, because the level is a small model's one-shot guess and guessing Low would truncate a thought mid-stream. MaxThinkingTokens was carried but, by its own comment, "no provider reads this today". Net effect: every chat turn got 8000 tokens of headroom and reasoned at whatever --reasoning-effort the server was launched with (medium on hive A). Trivial questions included. Effort is the right knob precisely because it sidesteps that concern: telling a model to think LESS cannot truncate it mid-thought, it just produces a shorter one. So the classifier level can safely govern effort even though it must not govern the token ceiling. Transport matters, and this was measured rather than assumed. Against rune (Qwen 3.8 on llama.cpp), one fixed prompt: top-level "reasoning_effort" low 421 / medium 416 / xhigh 421 chars chat_template_kwargs low 430 / xhigh 991 chars enable_thinking:false 0 chars (control) A top-level field is silently ignored. The first cut of this change sent exactly that and would have shipped a no-op. It now rides inside chat_template_kwargs alongside enable_thinking. Mapping (ThinkingBudget.Effort, overridable per level via [effort.thinking.] effort = "..."): off -> enable_thinking:false, no effort key low -> "low" medium -> "medium" high -> "medium", NOT xhigh — the level is a one-shot guess and xhigh is a large latency jump for a guess to be spending. Raise it in config if wanted. An empty Effort omits the key, so the server's launch default applies and backends without an effort dial are unaffected. --- .../internal/classifier/effort.go | 18 +++++++++++--- .../internal/classifier/from_config.go | 3 +++ familiar-gateway/internal/config/config.go | 4 ++++ familiar-gateway/internal/llm/openai.go | 24 +++++++++++++++++-- familiar-gateway/internal/llm/provider.go | 10 ++++++++ .../internal/pipeline/pipeline.go | 6 ++++- 6 files changed, 59 insertions(+), 6 deletions(-) diff --git a/familiar-gateway/internal/classifier/effort.go b/familiar-gateway/internal/classifier/effort.go index ecdf056..b3b303c 100644 --- a/familiar-gateway/internal/classifier/effort.go +++ b/familiar-gateway/internal/classifier/effort.go @@ -14,6 +14,14 @@ package classifier type ThinkingBudget struct { Enabled bool TokenBudget int + // Effort is the backend's native thinking-depth selector (Qwen 3.8: + // low/medium/xhigh). Empty leaves it unset so the server default + // applies. Unlike TokenBudget — which only grants headroom and is + // deliberately NOT allowed to bound the trusted path (see the comment + // at the thinkingHeadroom assignment in pipeline.go) — this tells the + // model to think less, which is safe: it cannot truncate a thought + // mid-stream, it just produces a shorter one. + Effort string } // MemoryBudget is the resolved retrieval shape for one MemoryDepth. @@ -52,9 +60,13 @@ func DefaultResolver() *EffortResolver { return &EffortResolver{ Thinking: map[ThinkingLevel]ThinkingBudget{ ThinkingOff: {Enabled: false}, - ThinkingLow: {Enabled: true, TokenBudget: 500}, - ThinkingMedium: {Enabled: true, TokenBudget: 2000}, - ThinkingHigh: {Enabled: true, TokenBudget: 8000}, + ThinkingLow: {Enabled: true, TokenBudget: 500, Effort: "low"}, + ThinkingMedium: {Enabled: true, TokenBudget: 2000, Effort: "medium"}, + // High maps to medium, not xhigh, on purpose. The level is a small + // model's one-shot guess, and xhigh is a large jump in latency for + // a guess to be spending. Operators can raise it per-level via + // [effort.thinking.high] effort = "xhigh". + ThinkingHigh: {Enabled: true, TokenBudget: 8000, Effort: "medium"}, }, Memory: map[MemoryDepth]MemoryBudget{ MemoryNone: {Skip: true}, diff --git a/familiar-gateway/internal/classifier/from_config.go b/familiar-gateway/internal/classifier/from_config.go index a7212b5..b339a93 100644 --- a/familiar-gateway/internal/classifier/from_config.go +++ b/familiar-gateway/internal/classifier/from_config.go @@ -33,6 +33,9 @@ func ResolverFromConfig(cfg config.EffortConfig) *EffortResolver { if src.TokenBudget > 0 { merged.TokenBudget = src.TokenBudget } + if src.Effort != "" { + merged.Effort = src.Effort + } r.Thinking[level] = merged } diff --git a/familiar-gateway/internal/config/config.go b/familiar-gateway/internal/config/config.go index 59e610b..f2799fc 100644 --- a/familiar-gateway/internal/config/config.go +++ b/familiar-gateway/internal/config/config.go @@ -113,6 +113,10 @@ type EffortThinkingConfig struct { type EffortThinkingLevel struct { Enabled *bool `toml:"enabled"` // nil = use level-default TokenBudget int `toml:"token_budget"` + // Effort overrides the backend's native thinking depth for this level + // (e.g. "low", "medium", "xhigh" on Qwen 3.8). Empty = use the + // level-default; set it to "" explicitly nowhere — omit the key instead. + Effort string `toml:"effort"` } // EffortMemoryDepthConfig — retrieval knobs per MemoryDepth. diff --git a/familiar-gateway/internal/llm/openai.go b/familiar-gateway/internal/llm/openai.go index 1c337ea..7b125bd 100644 --- a/familiar-gateway/internal/llm/openai.go +++ b/familiar-gateway/internal/llm/openai.go @@ -12,6 +12,26 @@ import ( "time" ) +// thinkingKwargs builds the chat_template_kwargs payload. +// +// reasoning_effort MUST travel inside chat_template_kwargs, not as a +// top-level request field. Measured against rune (Qwen 3.8 on llama.cpp) with +// one fixed prompt: as a top-level field, low/medium/xhigh all produced ~420 +// characters of reasoning — silently ignored. Inside chat_template_kwargs the +// same prompt gave 430 chars at low and 991 at xhigh. The server's +// --reasoning-effort launch flag sets the default; this overrides it per +// request. +// +// An empty Effort omits the key entirely, so the server default still applies +// and backends without an effort dial are unaffected. +func thinkingKwargs(req CompletionRequest) map[string]any { + kw := map[string]any{"enable_thinking": req.EnableThinking} + if req.EnableThinking && req.ReasoningEffort != "" { + kw["reasoning_effort"] = req.ReasoningEffort + } + return kw +} + // OpenAIProvider implements Provider for OpenAI-compatible endpoints // (llama-server, Ollama, vLLM, etc.). type OpenAIProvider struct { @@ -322,7 +342,7 @@ func (p *OpenAIProvider) Complete(ctx context.Context, req CompletionRequest) (* // to Qwen3.5-122B, hardcoding true forced thinking on everywhere, // which broke tier 3's "fast structured output, no thinking" // contract. Respect the request field now. See BUGS.md Bug 2. - ChatTemplateKwargs: map[string]any{"enable_thinking": req.EnableThinking}, + ChatTemplateKwargs: thinkingKwargs(req), } if req.ToolChoice != "" { body.ToolChoice = req.ToolChoice @@ -417,7 +437,7 @@ func (p *OpenAIProvider) CompleteStream(ctx context.Context, req CompletionReque Tools: tools, // See Complete() for why this respects req.EnableThinking rather // than forcing true. Same rationale applies for streaming. - ChatTemplateKwargs: map[string]any{"enable_thinking": req.EnableThinking}, + ChatTemplateKwargs: thinkingKwargs(req), } if req.ToolChoice != "" { body.ToolChoice = req.ToolChoice diff --git a/familiar-gateway/internal/llm/provider.go b/familiar-gateway/internal/llm/provider.go index 82a1a78..f3dd2a0 100644 --- a/familiar-gateway/internal/llm/provider.go +++ b/familiar-gateway/internal/llm/provider.go @@ -61,6 +61,16 @@ type CompletionRequest struct { MaxTokens int Stream bool EnableThinking bool + // ReasoningEffort selects the model's native thinking depth when the + // backend supports it (Qwen 3.8 exposes low/medium/xhigh). Empty means + // "do not send it" — the server's launch default then applies, which is + // what happened for every turn before this existed. + // + // This is the knob EnableThinking/MaxThinkingTokens could not provide. + // MaxThinkingTokens only ever widened max_tokens headroom; it never told + // the model to think less, so a trivial question still reasoned at + // whatever depth the server was started with. + ReasoningEffort string // MaxThinkingTokens is a soft budget for the model's reasoning tokens. // Zero means "no explicit budget" — providers pick their own default. // Providers that don't expose a thinking budget ignore this field. diff --git a/familiar-gateway/internal/pipeline/pipeline.go b/familiar-gateway/internal/pipeline/pipeline.go index c089ca9..cc2a3fe 100644 --- a/familiar-gateway/internal/pipeline/pipeline.go +++ b/familiar-gateway/internal/pipeline/pipeline.go @@ -2502,7 +2502,11 @@ func (p *Pipeline) buildLLMRequest(messages []llm.Message, route *routeResult, i // headroom we really granted rather than the level's nominal // budget, for whenever a provider does gain a separate knob. MaxThinkingTokens: thinkingHeadroom, - OnReasoningChunk: onReasoningChunk, + // The classifier's level now actually reaches the model. Previously + // it only widened max_tokens headroom, so every turn reasoned at the + // server's launch default regardless of how trivial the question was. + ReasoningEffort: thinkingBudget.Effort, + OnReasoningChunk: onReasoningChunk, } if overrides != nil { req.Temperature = overrides.Temperature From 1dc356d93be89353a6b4b6979853ee3b46bbf75d Mon Sep 17 00:00:00 2001 From: sixvolts Date: Mon, 7 Sep 2026 20:53:20 +0000 Subject: [PATCH 2/2] test(llm): pin reasoning_effort transport and its guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured against rune (Qwen 3.8 / llama.cpp): a top-level reasoning_effort field is silently ignored — low/medium/xhigh all gave ~420 chars of reasoning on one fixed prompt — while inside chat_template_kwargs the same prompt gave 430 at low and 991 at xhigh. Moving it back to a top-level field would break nothing at runtime and just quietly stop working, so the test asserts BOTH that it rides in chat_template_kwargs and that it is absent as a top-level field. Also pins the guard that effort only ships when thinking is on, so backends with no effort dial (MLX/Gemma on the tier-4 path) keep receiving exactly what they got before. Verified live that MLX returns HTTP 200 and ignores the key rather than erroring, so this change cannot break tier 4. --- familiar-gateway/internal/llm/openai_test.go | 77 ++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/familiar-gateway/internal/llm/openai_test.go b/familiar-gateway/internal/llm/openai_test.go index b655cbf..fccc180 100644 --- a/familiar-gateway/internal/llm/openai_test.go +++ b/familiar-gateway/internal/llm/openai_test.go @@ -476,3 +476,80 @@ func TestOpenAIProviderCompleteStream_UsageWinsOverPredictedN(t *testing.T) { t.Errorf("DecodeMs = %v, want 4000", resp.DecodeMs) } } + +// TestOpenAIComplete_ReasoningEffort pins the TRANSPORT of the effort knob, +// which is the part that fails invisibly. +// +// Measured against rune (Qwen 3.8 on llama.cpp): a TOP-LEVEL "reasoning_effort" +// request field is silently ignored — low/medium/xhigh all produced ~420 chars +// of reasoning on one fixed prompt. Inside chat_template_kwargs the same prompt +// gave 430 chars at low and 991 at xhigh. A change moving this back to a +// top-level field would break nothing at runtime; it would just quietly stop +// working. Hence this test. +// +// Also pins the guard: effort only rides along when thinking is on, so a +// backend without an effort dial (MLX/Gemma on the tier-4 path) keeps +// receiving exactly what it received before. Verified live that MLX returns +// HTTP 200 and ignores the key rather than erroring. +func TestOpenAIComplete_ReasoningEffort(t *testing.T) { + capture := func(t *testing.T, req CompletionRequest) map[string]any { + t.Helper() + var body struct { + ChatTemplateKwargs map[string]any `json:"chat_template_kwargs"` + ReasoningEffort string `json:"reasoning_effort"` + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&body) + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) + })) + defer srv.Close() + + p := NewOpenAIProvider("test", srv.URL, "") + if _, err := p.Complete(context.Background(), req); err != nil { + t.Fatalf("Complete: %v", err) + } + if body.ReasoningEffort != "" { + t.Errorf("reasoning_effort must NOT be a top-level field (the server ignores it there), got %q", + body.ReasoningEffort) + } + return body.ChatTemplateKwargs + } + + t.Run("forwarded inside chat_template_kwargs when thinking is on", func(t *testing.T) { + kw := capture(t, CompletionRequest{ + Model: "m", + Messages: []Message{{Role: "user", Content: "hi"}}, + EnableThinking: true, + ReasoningEffort: "low", + }) + if got, ok := kw["reasoning_effort"]; !ok || got != "low" { + t.Errorf("expected chat_template_kwargs.reasoning_effort=low, got %+v", kw) + } + }) + + t.Run("omitted when thinking is off", func(t *testing.T) { + kw := capture(t, CompletionRequest{ + Model: "m", + Messages: []Message{{Role: "user", Content: "hi"}}, + EnableThinking: false, + ReasoningEffort: "xhigh", + }) + if _, ok := kw["reasoning_effort"]; ok { + t.Errorf("reasoning_effort must not be sent when thinking is off, got %+v", kw) + } + }) + + t.Run("omitted when unset so the server default applies", func(t *testing.T) { + kw := capture(t, CompletionRequest{ + Model: "m", + Messages: []Message{{Role: "user", Content: "hi"}}, + EnableThinking: true, + }) + if _, ok := kw["reasoning_effort"]; ok { + t.Errorf("reasoning_effort must be omitted when unset, got %+v", kw) + } + if got, ok := kw["enable_thinking"]; !ok || got != true { + t.Errorf("enable_thinking must still be forwarded, got %+v", kw) + } + }) +}