From 76a944c7daa8b3d5f5c3d2ef8bd7c40c206ae217 Mon Sep 17 00:00:00 2001 From: rezaho Date: Mon, 3 Aug 2026 00:53:48 +0200 Subject: [PATCH 1/2] feat(models): a caller can keep the cache breakpoint off per-request rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A prompt-cache entry is only worth writing if the NEXT request can read it, which requires its hashed prefix to be bytes that request still contains. When a caller appends per-request content after the durable conversation — a clock, a budget figure, anything derived from "now" — the unconditional last-message breakpoint lands on exactly the row the next request cannot reproduce, so every request writes a fresh entry and reads none. `CACHE_EXEMPT_KEY` is a neutral per-item key on the caller's own message dict (the `defer_loading` shape) marking a row as per-request. The marker then lands on the last durable row, which the next request contains by construction since durable rows only ever grow by append. Counted from the end and stopping at the first unmarked row: the exemption is about position, so a marked row with durable rows after it is not a tail. An all-exempt list writes no marker at all. Measured on Bedrock/Opus 5, single-step turns with tools present: marker on the volatile row: turn 1 write=8211 -> turn 2 write=8226 read=0 marker on the last durable row: turn 1 write=8425 -> turn 2 write=15 read=8425 With no exempt row the payload is byte-identical to before, so no existing caller changes behaviour. Twinned on the OAuth leg, which is not a subclass and has its own payload builder. The key never reaches the wire — both builders rebuild rows with only the wire-legal fields. --- src/marsys/models/adapters/anthropic.py | 48 ++++++- src/marsys/models/adapters/anthropic_oauth.py | 12 +- tests/models/test_prompt_cache_breakpoint.py | 127 ++++++++++++++++++ 3 files changed, 183 insertions(+), 4 deletions(-) diff --git a/src/marsys/models/adapters/anthropic.py b/src/marsys/models/adapters/anthropic.py index e3af976a..d480a4eb 100644 --- a/src/marsys/models/adapters/anthropic.py +++ b/src/marsys/models/adapters/anthropic.py @@ -86,11 +86,35 @@ def _anthropic_model_requires_adaptive_thinking(model_name: str) -> bool: CACHE_CONTROL_EPHEMERAL = {"type": "ephemeral"} - -def mark_conversation_tail_for_cache(messages: List[Dict[str, Any]]) -> None: +# A caller marks a message row with this key to say "my content here changes every +# request; do not put the cache breakpoint on me". Neutral and per-item, riding the +# caller's own message dict — the `defer_loading` shape, which is this codebase's +# established way for a caller to signal request structure without a new request +# parameter. Stripped during conversion; it never reaches the wire. +# +# Why a caller needs this: the breakpoint's value is that the NEXT request can read +# the entry this one writes, which requires the entry's hashed prefix to consist of +# bytes the next request still contains. A row whose text is regenerated per request +# (a clock, a budget figure, anything derived from "now") is by construction absent +# from the next request, so an entry written at or after it is unreadable forever — +# each turn writes a fresh entry and reads none. Measured on Bedrock/Opus 5, single- +# step turns, tools present: marker on the volatile row → turn 2 `read=0`; marker on +# the last durable row → turn 2 `read=8425`. +CACHE_EXEMPT_KEY = "cache_exempt" + + +def mark_conversation_tail_for_cache( + messages: List[Dict[str, Any]], *, volatile_tail: int = 0 +) -> None: """Place ONE prompt-cache breakpoint on the last content block of the last message, in place on ``messages`` — the platform's multi-turn caching pattern. + ``volatile_tail`` excludes that many trailing messages from carrying the marker, + for a caller that appends per-request content after the durable conversation (see + ``CACHE_EXEMPT_KEY``). The marker then lands on the last DURABLE row, which the + next request still contains verbatim, so the entry stays readable. Default 0 keeps + the payload byte-identical to the unparameterized form for every other caller. + Adapter-owned and unconditional, matching the only other `cache_control` site in this codebase (the OAuth adapter's static Claude-Code prefix block). Only the payload builder knows the rendered block layout, and caching is prefix-match @@ -118,6 +142,11 @@ def mark_conversation_tail_for_cache(messages: List[Dict[str, Any]]) -> None: the model's cacheable minimum silently writes nothing and costs nothing, so no size check is needed here. """ + if volatile_tail: + # Step back past the per-request rows. Every row is volatile (a caller that + # marked the whole list) → nothing durable to anchor an entry to, so no marker: + # writing one would cost a fresh entry per request and read none. + messages = messages[:-volatile_tail] if not messages: return last = messages[-1] @@ -317,6 +346,19 @@ def format_request_payload(self, messages: List[Dict], **kwargs) -> Dict[str, An system_message = None user_messages = [] + # How many TRAILING rows the caller marked as per-request (``CACHE_EXEMPT_KEY``), + # so the breakpoint below lands on the last durable row instead. Counted from the + # end and stopping at the first unmarked row: the exemption is about position (what + # the next request will still contain), so a marked row with durable rows after it + # is not a tail and does not shift the marker. Each of these converts 1:1 into + # ``user_messages``, so the count carries over. The key itself never reaches the + # wire — the regular-message branch rebuilds rows with only role/content. + volatile_tail = 0 + for msg in reversed(messages): + if not msg.get(CACHE_EXEMPT_KEY): + break + volatile_tail += 1 + for msg in messages: if msg.get("role") == "system": system_message = msg.get("content") @@ -533,7 +575,7 @@ def format_request_payload(self, messages: List[Dict], **kwargs) -> Dict[str, An # final message, and the json-mode fallback above may still append a hint # block there. Placing it after every content mutation is what makes "the # tail" mean the actual tail. - mark_conversation_tail_for_cache(user_messages) + mark_conversation_tail_for_cache(user_messages, volatile_tail=volatile_tail) return payload diff --git a/src/marsys/models/adapters/anthropic_oauth.py b/src/marsys/models/adapters/anthropic_oauth.py index d78f4c12..cab6add6 100644 --- a/src/marsys/models/adapters/anthropic_oauth.py +++ b/src/marsys/models/adapters/anthropic_oauth.py @@ -7,6 +7,7 @@ from typing import Any, Dict, List, Optional from marsys.models.adapters.anthropic import ( + CACHE_EXEMPT_KEY, _anthropic_model_rejects_temperature, _anthropic_model_requires_adaptive_thinking, mark_conversation_tail_for_cache, @@ -443,6 +444,15 @@ def format_request_payload(self, messages: List[Dict], **kwargs) -> Dict[str, An system_message = None converted_messages = [] + # Trailing per-request rows the caller exempted from the breakpoint — twinned with + # the api-key adapter (this class is not a subclass of it, so the two payload + # builders are kept deliberately parallel). See ``CACHE_EXEMPT_KEY``. + volatile_tail = 0 + for msg in reversed(messages): + if not msg.get(CACHE_EXEMPT_KEY): + break + volatile_tail += 1 + for msg in messages: role = msg.get("role") content = msg.get("content") @@ -611,7 +621,7 @@ def format_request_payload(self, messages: List[Dict], **kwargs) -> Dict[str, An # in an OAuth payload — the static Claude-Code prefix block in # ``_build_system_array`` is the first — which keeps the payload two under # the API's four-breakpoint ceiling. - mark_conversation_tail_for_cache(converted_messages) + mark_conversation_tail_for_cache(converted_messages, volatile_tail=volatile_tail) return payload diff --git a/tests/models/test_prompt_cache_breakpoint.py b/tests/models/test_prompt_cache_breakpoint.py index e7366d91..85008d27 100644 --- a/tests/models/test_prompt_cache_breakpoint.py +++ b/tests/models/test_prompt_cache_breakpoint.py @@ -22,6 +22,7 @@ import pytest from marsys.models.adapters.anthropic import ( + CACHE_EXEMPT_KEY, AnthropicAdapter, mark_conversation_tail_for_cache, ) @@ -502,3 +503,129 @@ def test_the_marker_lands_after_the_json_mode_hint_not_before_it(): assert "JSON" in json.dumps(content) assert content[-1]["cache_control"] == EPHEMERAL assert sum(1 for b in content if "cache_control" in b) == 1 + + +# ── the per-request-content exemption (CACHE_EXEMPT_KEY) ───────────────────────────── +# +# A caller that appends per-request content after the durable conversation (a clock, a +# budget figure — anything derived from "now") needs the breakpoint to stay on the last +# DURABLE row. The entry's value is that the NEXT request can read it, which requires the +# hashed prefix to be bytes the next request still contains; a regenerated row is absent +# from it by construction, so an entry written at or after that row is unreadable forever. +# Measured on Bedrock/Opus 5, single-step turns with tools present: marker on the volatile +# row -> turn 2 read=0; marker on the last durable row -> turn 2 read=8425. + + +def _marked_indices(payload) -> list[int]: + out = [] + for i, msg in enumerate(payload["messages"]): + content = msg.get("content") + blocks = content if isinstance(content, list) else [] + if any(isinstance(b, dict) and b.get("cache_control") for b in blocks): + out.append(i) + return out + + +def test_an_exempt_tail_row_moves_the_marker_to_the_last_durable_row(): + payload = _api().format_request_payload( + [ + {"role": "user", "content": "durable question"}, + {"role": "assistant", "content": "durable answer"}, + {"role": "user", "content": "now: 09:00", CACHE_EXEMPT_KEY: True}, + ] + ) + assert _marked_indices(payload) == [1], "the marker must sit on the last durable row" + assert payload["messages"][2]["content"] == "now: 09:00", "the row still reaches the model" + + +def test_several_exempt_trailing_rows_are_all_stepped_past(): + payload = _api().format_request_payload( + [ + {"role": "user", "content": "durable"}, + {"role": "user", "content": "volatile a", CACHE_EXEMPT_KEY: True}, + {"role": "user", "content": "volatile b", CACHE_EXEMPT_KEY: True}, + ] + ) + assert _marked_indices(payload) == [0] + + +def test_an_exempt_row_with_durable_rows_after_it_does_not_move_the_marker(): + """The exemption is about POSITION — what the next request will still contain. A marked + row that is not part of the trailing run is not a tail, so the marker stays at the end.""" + payload = _api().format_request_payload( + [ + {"role": "user", "content": "volatile", CACHE_EXEMPT_KEY: True}, + {"role": "user", "content": "durable"}, + ] + ) + assert _marked_indices(payload) == [1] + + +def test_an_all_exempt_list_writes_no_marker_at_all(): + """Nothing durable to anchor an entry to: a marker would cost a fresh entry per request + and read none, so none is written.""" + payload = _api().format_request_payload( + [{"role": "user", "content": "volatile", CACHE_EXEMPT_KEY: True}] + ) + assert _marked_indices(payload) == [] + + +def test_the_exemption_key_never_reaches_the_wire(): + payload = _api().format_request_payload( + [ + {"role": "user", "content": "durable"}, + {"role": "user", "content": "volatile", CACHE_EXEMPT_KEY: True}, + ] + ) + assert CACHE_EXEMPT_KEY not in json.dumps(payload) + + +def test_the_no_key_path_is_byte_identical_to_the_unparameterized_form(): + """Every other framework caller must be unaffected: with no exempt row the payload is + exactly what it was before this parameter existed.""" + messages = [ + {"role": "user", "content": "one"}, + {"role": "assistant", "content": "two"}, + {"role": "user", "content": "three"}, + ] + payload = _api().format_request_payload([dict(m) for m in messages]) + baseline = [dict(m) for m in messages] + mark_conversation_tail_for_cache(baseline) # the default, volatile_tail=0 + assert payload["messages"] == baseline + + +def test_oauth_leg_honors_the_exemption_too(): + """The OAuth adapter is not a subclass, so its payload builder is kept deliberately + twinned. Its static Claude-Code prefix block is the other marker.""" + payload = _oauth().format_request_payload( + [ + {"role": "user", "content": "durable"}, + {"role": "user", "content": "volatile", CACHE_EXEMPT_KEY: True}, + ] + ) + assert _marked_indices(payload) == [0] + assert CACHE_EXEMPT_KEY not in json.dumps(payload) + + +def test_bedrock_inherits_the_exemption(): + """Bedrock does not override ``format_request_payload``, and it is the production leg + the measured figures come from.""" + payload = BedrockAdapter( + model_name="claude-opus-5", api_key="tok" + ).format_request_payload( + [ + {"role": "user", "content": "durable"}, + {"role": "user", "content": "volatile", CACHE_EXEMPT_KEY: True}, + ] + ) + assert _marked_indices(payload) == [0] + + +def test_the_exemption_is_deterministic_and_idempotent(): + messages = [ + {"role": "user", "content": "durable"}, + {"role": "user", "content": "volatile", CACHE_EXEMPT_KEY: True}, + ] + first = _api().format_request_payload([dict(m) for m in messages]) + second = _api().format_request_payload([dict(m) for m in messages]) + assert json.dumps(first, sort_keys=True) == json.dumps(second, sort_keys=True) From f82d1a34c609e767350a7f7f728b0432e90bfd2e Mon Sep 17 00:00:00 2001 From: rezaho Date: Fri, 7 Aug 2026 15:12:29 +0200 Subject: [PATCH 2/2] fix(anthropic-oauth): clamp fixed thinking budgets under max_tokens Parity with AnthropicAdapter._thinking_payload: budget_tokens >= max_tokens is an illegal payload (live 400) for every non-adaptive model, and a thinking flag without a usable budget put budget_tokens=None/0 on the wire. Clamp with the twin's headroom rules, drop thinking when nothing legal fits, warn either way. The shape that surfaced it: a background model built at max_tokens=4096 with the default 8192 thinking budget 400s on every call once its model id resolves to a non-adaptive catalog model. --- src/marsys/models/adapters/anthropic_oauth.py | 33 ++++++++++++++++++- .../test_oauth_claude5_payload_shape.py | 25 ++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/marsys/models/adapters/anthropic_oauth.py b/src/marsys/models/adapters/anthropic_oauth.py index cab6add6..8111bb3e 100644 --- a/src/marsys/models/adapters/anthropic_oauth.py +++ b/src/marsys/models/adapters/anthropic_oauth.py @@ -53,6 +53,12 @@ class AnthropicOAuthAdapter(APIProviderAdapter): # Enable streaming mode - Claude OAuth uses SSE streaming streaming = True + # Anthropic's documented bounds for fixed-budget thinking: budget_tokens >= 1024 and + # strictly less than max_tokens (thinking spends from the same output allowance). + # Mirrors AnthropicAdapter — the two payload builders are kept deliberately parallel. + _THINKING_MIN_BUDGET = 1024 + _THINKING_HEADROOM = 1024 + # CRITICAL: Exact prefix required - no trailing characters! CLAUDE_CODE_PREFIX = "You are Claude Code, Anthropic's official CLI for Claude." @@ -540,8 +546,33 @@ def format_request_payload(self, messages: List[Dict], **kwargs) -> Dict[str, An effort = kwargs.get("reasoning_effort") if effort: payload.setdefault("output_config", {})["effort"] = str(effort).lower() + elif not isinstance(budget, int) or budget <= 0: + # Thinking requested without a usable fixed budget: sending a null/zero + # budget_tokens is an illegal payload, so thinking is dropped instead. + logger.warning( + "thinking requested without a usable budget (budget=%r); " + "thinking disabled for this call", budget, + ) else: - payload["thinking"] = {"type": "enabled", "budget_tokens": budget} + # Clamp under max_tokens (parity with AnthropicAdapter._thinking_payload): + # the API 400s on budget_tokens >= max_tokens, and headroom keeps a usable + # text/tool allowance after a maximally-thinky step. A budget that cannot + # fit disables thinking for the call rather than failing it. + max_allowed = payload["max_tokens"] + clamped = min(budget, max_allowed - self._THINKING_HEADROOM) + if clamped < self._THINKING_MIN_BUDGET: + logger.warning( + "thinking_budget=%s cannot fit under max_tokens=%s " + "(min budget %s + headroom); thinking disabled for this call", + budget, max_allowed, self._THINKING_MIN_BUDGET, + ) + else: + if clamped < budget: + logger.warning( + "thinking_budget=%s clamped to %s to fit under max_tokens=%s", + budget, clamped, max_allowed, + ) + payload["thinking"] = {"type": "enabled", "budget_tokens": clamped} # Temperature only when the model accepts it AND thinking is off — the # reasoning-capable models 400 on the key ("`temperature` is deprecated diff --git a/tests/models/test_oauth_claude5_payload_shape.py b/tests/models/test_oauth_claude5_payload_shape.py index 45d8bfbf..bc9472ca 100644 --- a/tests/models/test_oauth_claude5_payload_shape.py +++ b/tests/models/test_oauth_claude5_payload_shape.py @@ -73,3 +73,28 @@ def test_short_aliases_resolve_and_shape_as_claude5(): ) assert payload["thinking"] == {"type": "adaptive"}, alias assert "temperature" not in payload, alias + + +def test_fixed_budget_is_clamped_under_max_tokens(): + """Parity with the api-key twin: budget_tokens >= max_tokens is a live 400. + The shape that mattered: a background model built at max_tokens=4096 with the + default 8192 thinking budget — every call was an illegal payload on this leg.""" + adapter = _oauth("claude-haiku-4-5-20251001", budget=8192) + adapter.max_tokens = 4096 + payload = adapter.format_request_payload(MESSAGES, thinking_budget=8192) + assert payload["thinking"] == {"type": "enabled", "budget_tokens": 3072} + + +def test_budget_that_cannot_fit_disables_thinking(): + adapter = _oauth("claude-haiku-4-5-20251001", budget=8192) + adapter.max_tokens = 1536 # headroom leaves less than the documented minimum budget + payload = adapter.format_request_payload(MESSAGES, thinking_budget=8192) + assert "thinking" not in payload + + +def test_thinking_flag_without_budget_sends_no_null_budget(): + """enable_thinking with no usable budget used to put budget_tokens=None/0 on the + wire; thinking is dropped instead of sending an illegal payload.""" + adapter = _oauth("claude-haiku-4-5-20251001", enable=True, budget=0) + payload = adapter.format_request_payload(MESSAGES) + assert "thinking" not in payload