From 6241178a7550e5a0ffadd85f0eb04af0160f9988 Mon Sep 17 00:00:00 2001 From: ascorb12 Date: Sun, 23 Aug 2026 20:52:10 -0400 Subject: [PATCH] fix(scheduler): clamp admission to the actual KV pool A prompt that fits the model's advertised max_seq_len but exceeds the allocated KV pool passes admission and is queued forever with no error, no log line, and an idle engine (issue #111). Clamp the admission bound to num_pages * page_size and reuse the existing too-long rejection path so such prompts fail loudly with context_length_exceeded instead. Co-Authored-By: Claude Fable 5 --- python/freetoken/scheduler/scheduler.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/python/freetoken/scheduler/scheduler.py b/python/freetoken/scheduler/scheduler.py index 35541161..38bf8779 100644 --- a/python/freetoken/scheduler/scheduler.py +++ b/python/freetoken/scheduler/scheduler.py @@ -487,10 +487,17 @@ def _process_one_msg(self, msg: BaseBackendMsg) -> None: ) return input_len, max_seq_len = len(msg.input_ids), self.engine.max_seq_len - max_output_len = max_seq_len - input_len + # max_seq_len is the model's advertised context, which can far exceed the + # KV pool actually allocated (see issue #111): a prompt that passes this + # check but can never be granted enough pages is queued forever with no + # error and no log line. Clamp admission to the real pool so oversized + # prompts fail loudly with the same error clients already understand. + pool_tokens = self.engine.num_pages * self.config.page_size + effective_max = min(max_seq_len, pool_tokens) + max_output_len = effective_max - input_len if max_output_len <= 0: logger.warning_rank0( - f"Input sequence length {input_len} exceeds {max_seq_len}, " + f"Input sequence length {input_len} exceeds {effective_max}, " f"request {msg.uid} is dropped." ) # Tell the client instead of dropping silently — otherwise its wait_for_ack @@ -502,7 +509,7 @@ def _process_one_msg(self, msg: BaseBackendMsg) -> None: # "prompt is too long: N tokens > M" is the phrasing Claude Code and # OpenClaw match on; the Anthropic wire has no error code to read. error=( - f"prompt is too long: {input_len} tokens > {max_seq_len} maximum " + f"prompt is too long: {input_len} tokens > {effective_max} maximum " f"(prompt + generation); shorten the prompt or increase the KV " f"cache budget" ),