From 8e271c55dbf1429c439177698d3f395ce65585ca Mon Sep 17 00:00:00 2001 From: bahtyar Date: Thu, 14 May 2026 18:31:02 +0800 Subject: [PATCH] fix(providers): auto-inflate max_tokens for reasoning models Reasoning models like glm-5-turbo consume max_tokens budget with thinking tokens, leaving no room for actual content. With max_tokens=4096 the model thinks for 127s and produces 0 chars of output. Auto-inflate to at least 16384 for known reasoning models, matching hermes's approach of max(user_config, reasoning_budget + output_budget). Closes #378 Bahtya --- crates/kestrel-providers/src/openai_compat.rs | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/crates/kestrel-providers/src/openai_compat.rs b/crates/kestrel-providers/src/openai_compat.rs index 8796ec2..cffba3f 100644 --- a/crates/kestrel-providers/src/openai_compat.rs +++ b/crates/kestrel-providers/src/openai_compat.rs @@ -136,7 +136,12 @@ impl OpenAiCompatProvider { }); if let Some(max_tokens) = request.max_tokens { - body["max_tokens"] = json!(max_tokens); + let effective = if is_reasoning_model(&request.model) { + max_tokens.max(16384) + } else { + max_tokens + }; + body["max_tokens"] = json!(effective); } if let Some(temp) = request.temperature { body["temperature"] = json!(temp); @@ -381,6 +386,19 @@ fn build_openai_tool_call_deltas( Some(deltas) } +/// Check if a model is a known reasoning model that uses thinking tokens. +/// +/// Reasoning models consume `max_tokens` budget with thinking/reasoning tokens. +/// We auto-inflate the budget so thinking doesn't crowd out actual content. +fn is_reasoning_model(model: &str) -> bool { + let m = model.to_lowercase(); + m.contains("glm-5") + || m.starts_with("deepseek-r") + || m.starts_with("o1") + || m.starts_with("o3") + || m.starts_with("o4") +} + #[derive(Debug, Deserialize)] struct OpenAiResponse { choices: Vec,