diff --git a/docs/story-evaluation.md b/docs/story-evaluation.md index 46db1e0..43932d7 100644 --- a/docs/story-evaluation.md +++ b/docs/story-evaluation.md @@ -8,27 +8,31 @@ Scout, the production database, or Discord. The prepared screen is 48 writer calls (four models, six cases, two repeats) and 48 reviewer calls (two models, twelve calibration fixtures, two repeats). Each call reserves 32,768 input tokens and either 16,384 writer tokens or 8,192 -reviewer tokens. Reasoning tokens count against the output ceiling. The current -per-call ceilings total $27.60, so the proposed initial allowance is 96 calls -and $28. Up to two selected writer/reviewer pairings can later use 96 calls and -$64. A separately reviewed current-evidence acceptance packet can use four more -calls and $2.60. The exact reserved total is $92.60, with a proposed $100 outer -envelope for 196 calls. These are approval bounds, not expected spend. +reviewer tokens. Reasoning tokens count against the output ceiling. Input uses +the greater of the base price and the total price including possible cache +writes, even when the request asks the provider not to cache. The current +per-call ceilings total $30.00 for the prepared 96-call screen. These are +reservation bounds, not expected spend or an approved allowance. + +The original proposal reserved $92.60 within a $100 envelope for 196 calls. +That proposal predates cache-write-aware pricing. It remains historical evidence; +new plans and approvals must use the current calculation without rewriting old +ledgers or transferring unused allowances. The initial screen uses separate immutable ledgers, so the operator must track the aggregate approval across them: | Run | Calls | Per-run ceiling | | --- | ---: | ---: | -| Astra writer | 12 | $13.80 | -| Sol writer | 12 | $2.76 | -| Sonnet writer | 12 | $2.76 | +| Astra writer | 12 | $14.76 | +| Sol writer | 12 | $3.00 | +| Sonnet writer | 12 | $3.00 | | Gemini writer | 12 | $1.08 | -| Sol reviewer | 24 | $3.60 | -| Sonnet reviewer | 24 | $3.60 | +| Sol reviewer | 24 | $4.08 | +| Sonnet reviewer | 24 | $4.08 | -The worst-case selected pairing is 48 calls and $31.20 when Astra writes and -Sol or Sonnet reviews. Two such pairing ledgers reserve 96 calls and $62.40. +The worst-case selected pairing is 48 calls and $33.60 when Astra writes and +Sol or Sonnet reviews. Two such pairing ledgers reserve 96 calls and $67.20. No single ledger enforces the aggregate allowance across other artifact directories. @@ -61,7 +65,7 @@ runtime and is not written to artifacts. OPENROUTER_API_KEY=... python -m roger.story_eval run \ --stage writer --model sol --case straight_release --repeats 2 \ --artifact-dir /tmp/roger-story-eval-sol \ - --max-calls 2 --max-cost-usd 0.46 --approve-paid-calls + --max-calls 2 --max-cost-usd 0.50 --approve-paid-calls ``` Reviewer calibration uses `--stage reviewer --model sol`; each `--case` value @@ -103,8 +107,28 @@ explicit unresolved limit. The repository's regressions are derived synthetic reproductions. Exact captured outputs stay in their immutable evaluation bundle. -The API cache is disabled so repeated requests remain real repeats. Artifacts -contain the complete visible response text, an allowlisted routing-metadata +Response caching is disabled with `X-OpenRouter-Cache: false`. Provider prompt +caching is separate. OpenAI profiles request it off using +`prompt_cache_options: {"mode": "explicit"}` without breakpoints. Other profiles +record `provider_default`. Messages must be plain text, without explicit cache +markers or TTLs that would invalidate the price bound. Sonnet's input bound +covers default five-minute writes. Gemini implicit caching adds no write/storage +charge, so its total input price remains the base price; the catalog's explicit +storage surcharge is not a total input-token price. Missing or invalid cache +policy/pricing rejects the profile before a request can be sent. + +These settings follow the [provider caching documentation](https://openrouter.ai/docs/guides/best-practices/prompt-caching). +SDK tests verify serialization, not live endpoint acceptance of the new control. +Positive reported prompt-cache activity under requested-off stops the run after +recording the reported cost. Missing counters remain unknown; two zero counters +mean `none_reported`, not independently verified absence of caching. + +Artifacts retain allowlisted IDs in `response.generation_id`, and cache-read/write +token counts and upstream cost components in `response.usage_details`, even when +required usage is missing or the response is malformed. Unknown or invalid optional fields +stay `null` and do not replace a valid provider-reported charge. A cost difference +alone does not establish cache activity. Earlier artifacts are not rewritten. +Artifacts also contain the complete visible response text, an allowlisted routing-metadata view, token/cost accounting, sanitized error classes or status codes, and the exact messages, schema and non-secret request settings needed to reproduce the call. They do not contain the key, authorization headers, hidden reasoning or diff --git a/evals/stories/models.json b/evals/stories/models.json index 2cc4cf0..8070a2a 100644 --- a/evals/stories/models.json +++ b/evals/stories/models.json @@ -11,12 +11,13 @@ "provider": "openai", "provider_display_name": "OpenAI", "service_tier": "default", - "pricing_usd_per_million": {"prompt": 10, "completion": 50}, + "pricing_usd_per_million": {"prompt": 10, "completion": 50, "prompt_cache_write": 12.5}, "reasoning_effort": "medium", "writer_max_tokens": 16384, "reviewer_max_tokens": 8192, "available": true, - "note": "Strong capability baseline." + "note": "Strong capability baseline.", + "prompt_cache": "disabled" }, "sol": { "model": "openai/gpt-6.1-sol-20260929", @@ -27,12 +28,13 @@ "provider": "openai", "provider_display_name": "OpenAI", "service_tier": "default", - "pricing_usd_per_million": {"prompt": 2, "completion": 10}, + "pricing_usd_per_million": {"prompt": 2, "completion": 10, "prompt_cache_write": 2.5}, "reasoning_effort": "medium", "writer_max_tokens": 16384, "reviewer_max_tokens": 8192, "available": true, - "note": "Balanced OpenAI candidate." + "note": "Balanced OpenAI candidate.", + "prompt_cache": "disabled" }, "sonnet": { "model": "anthropic/claude-sonnet-5.5-20260928", @@ -43,12 +45,13 @@ "provider": "anthropic", "provider_display_name": "Anthropic", "service_tier": "default", - "pricing_usd_per_million": {"prompt": 2, "completion": 10}, + "pricing_usd_per_million": {"prompt": 2, "completion": 10, "prompt_cache_write": 2.5}, "reasoning_effort": "medium", "writer_max_tokens": 16384, "reviewer_max_tokens": 8192, "available": true, - "note": "Cross-provider writing and judgment candidate." + "note": "Cross-provider writing and judgment candidate.", + "prompt_cache": "provider_default" }, "gemini": { "model": "google/gemini-3.8-flash-20260902", @@ -59,18 +62,23 @@ "provider": "google-ai-studio", "provider_display_name": "Google AI Studio", "service_tier": "default", - "pricing_usd_per_million": {"prompt": 0.75, "completion": 3.75}, + "pricing_usd_per_million": {"prompt": 0.75, "completion": 3.75, "prompt_cache_write": 0.75}, "reasoning_effort": "medium", "writer_max_tokens": 16384, "reviewer_max_tokens": 8192, "available": true, - "note": "Catalog endpoint status -2 is recorded but uninterpreted; recheck before paid run." + "note": "Catalog endpoint status -2 is recorded but uninterpreted; recheck before paid run.", + "prompt_cache": "provider_default" } }, "notes": [ "Every request pins one dated model and one first-party provider, disables fallback, and accepts only the dated ID or its verified canonical base ID.", - "Input is reserved at 32768 tokens per call. Output reservations include reasoning tokens and are rounded up to cents per call.", + "Input is reserved at 32768 tokens per call using the greater of base and total cache-write input price, including when prompt caching is requested off. Output reservations include reasoning tokens and are rounded up to cents per call.", "The Gemini provider choice must be rechecked before a paid run; google-vertex/global requires a separately recorded preflight choice.", - "The initial 96-call writer and reviewer matrix has a $28 proposed cap. Two selected pairings allow at most 96 more calls with a $64 proposed cap. Reserve four more calls for separately reviewed held-out current evidence; the exact reservation is 196 calls and $92.60 within a proposed $100 envelope." - ] + "Historical initial proposal: 196 calls and $92.60 within a $100 envelope. That proposal predates cache-write-aware reservations; use the current plan for new approvals, without rewriting earlier ledgers or allowances.", + "Response caching is disabled separately. OpenAI prompt caching is requested off with explicit mode and no breakpoints; other providers retain their default behavior. Missing cache usage is unknown, not proof of no caching.", + "prompt_cache_write is the total USD input price per million tokens when writes are possible, not a storage surcharge. Sonnet uses its default five-minute write price. Gemini implicit caching adds no write/storage charge, so its value equals base input pricing. Explicit cache markers and TTLs are not supported." + ], + "cache_policy_checked_at": "2026-10-29", + "cache_policy_source": "https://openrouter.ai/docs/guides/best-practices/prompt-caching" } diff --git a/roger/story_eval.py b/roger/story_eval.py index 99653cd..135d828 100644 --- a/roger/story_eval.py +++ b/roger/story_eval.py @@ -11,6 +11,7 @@ import json import os import pathlib +import re import sys import time from decimal import Decimal, InvalidOperation @@ -41,6 +42,8 @@ class ModelProfile: service_tier: str prompt_usd_per_million: Decimal completion_usd_per_million: Decimal + prompt_cache: str + prompt_cache_write_usd_per_million: Decimal reasoning_effort: str writer_max_tokens: int reviewer_max_tokens: int @@ -51,6 +54,11 @@ class ModelProfile: @classmethod def from_record(cls, name: str, record: dict[str, Any]) -> ModelProfile: pricing = record["pricing_usd_per_million"] + try: + prompt_cache = record["prompt_cache"] + cache_write_price = Decimal(str(pricing["prompt_cache_write"])) + except (KeyError, InvalidOperation, TypeError, ValueError) as exc: + raise EvaluationError(f"missing or invalid prompt-cache profile: {name}") from exc profile = cls( name=name, model=record["model"], @@ -59,6 +67,8 @@ def from_record(cls, name: str, record: dict[str, Any]) -> ModelProfile: service_tier=record["service_tier"], prompt_usd_per_million=Decimal(str(pricing["prompt"])), completion_usd_per_million=Decimal(str(pricing["completion"])), + prompt_cache=prompt_cache, + prompt_cache_write_usd_per_million=cache_write_price, reasoning_effort=record["reasoning_effort"], writer_max_tokens=int(record["writer_max_tokens"]), reviewer_max_tokens=int(record["reviewer_max_tokens"]), @@ -71,10 +81,13 @@ def from_record(cls, name: str, record: dict[str, Any]) -> ModelProfile: or profile.service_tier != "default" or profile.reasoning_effort != "medium" or profile.writer_max_tokens <= 0 or profile.reviewer_max_tokens <= 0 + or profile.prompt_cache not in ("disabled", "provider_default") + or (profile.prompt_cache == "disabled" and profile.provider != "openai") or any( not price.is_finite() or price <= 0 for price in ( - profile.prompt_usd_per_million, profile.completion_usd_per_million + profile.prompt_usd_per_million, profile.completion_usd_per_million, + profile.prompt_cache_write_usd_per_million, ) ) ): @@ -114,7 +127,10 @@ def reserved_cost_usd(self) -> Decimal: f"request needs {self.input_token_upper_bound} input tokens, above the " f"{_INPUT_TOKEN_RESERVE} token reservation" ) - prompt = Decimal(_INPUT_TOKEN_RESERVE) * self.profile.prompt_usd_per_million + # Cover possible billed writes even when the request asks the provider not to cache. + prompt = Decimal(_INPUT_TOKEN_RESERVE) * max( + self.profile.prompt_usd_per_million, self.profile.prompt_cache_write_usd_per_million + ) completion = Decimal(self.max_tokens) * self.profile.completion_usd_per_million estimated = (prompt + completion) / Decimal(1_000_000) return estimated.quantize(Decimal("0.01"), rounding="ROUND_CEILING") @@ -132,6 +148,11 @@ def inventory(self) -> dict[str, Any]: "provider_display_name": self.profile.provider_display_name, "service_tier": self.profile.service_tier, "reasoning_effort": self.profile.reasoning_effort, + "response_cache": "disabled", + "prompt_cache": self.profile.prompt_cache, + "prompt_cache_write_usd_per_million": str( + self.profile.prompt_cache_write_usd_per_million + ), "max_tokens": self.max_tokens, "timeout_seconds": 900 if "reviewer" in self.stage else 1200, "input_token_upper_bound": self.input_token_upper_bound, @@ -424,7 +445,13 @@ def total(rows: list[dict[str, Any]]) -> str: def _request_kwargs(spec: CallSpec) -> dict[str, Any]: - return { + if any( + not isinstance(message, dict) or set(message) != {"role", "content"} + or not isinstance(message["content"], str) + for message in spec.messages + ): + raise EvaluationError("evaluation messages must be plain text without cache controls") + kwargs = { "model": spec.profile.model, "messages": spec.messages, "max_tokens": spec.max_tokens, @@ -444,6 +471,9 @@ def _request_kwargs(spec: CallSpec) -> dict[str, Any]: "reasoning": {"effort": spec.profile.reasoning_effort, "exclude": True}, }, } + if spec.profile.prompt_cache == "disabled": + kwargs["extra_body"]["prompt_cache_options"] = {"mode": "explicit"} + return kwargs def _request_record(spec: CallSpec) -> dict[str, Any]: @@ -558,6 +588,8 @@ def _visible_response(response: Any) -> dict[str, Any]: if choice is not None and message is None: shape_error = "missing_message" visible = { + "generation_id": _generation_id(response), + "usage_details": _usage_details(response), "model": getattr(response, "model", None), "finish_reason": getattr(choice, "finish_reason", None), "content": getattr(message, "content", None), @@ -570,6 +602,45 @@ def _visible_response(response: Any) -> dict[str, Any]: return visible +def _generation_id(response: Any) -> str | None: + value = getattr(response, "id", None) + if isinstance(value, str) and re.fullmatch(r"gen-[A-Za-z0-9_-]{1,252}", value): + return value + return None + + +def _usage_details(response: Any) -> dict[str, Any]: + usage = getattr(response, "usage", None) + prompt_details = getattr(usage, "prompt_tokens_details", None) + cost_details = getattr(usage, "cost_details", None) + + def field(container: Any, key: str) -> Any: + return container.get(key) if isinstance(container, dict) else getattr(container, key, None) + + counts = {} + prompt_tokens = getattr(usage, "prompt_tokens", None) + for key in ("cached_tokens", "cache_write_tokens"): + value = field(prompt_details, key) + counts[key] = value if ( + type(value) is int and value >= 0 + and (type(prompt_tokens) is not int or prompt_tokens < 0 or value <= prompt_tokens) + ) else None + costs = {} + for key in ( + "upstream_inference_cost", "upstream_inference_prompt_cost", + "upstream_inference_completions_cost", + ): + value = field(cost_details, key) + try: + amount = Decimal(str(value)) + except (InvalidOperation, TypeError, ValueError): + amount = None + costs[key] = ( + str(amount) if amount is not None and amount.is_finite() and amount >= 0 else None + ) + return {**counts, "cost_details": costs} + + def _usage(response: Any) -> dict[str, int | Decimal | None] | None: usage = getattr(response, "usage", None) if usage is None: @@ -654,10 +725,11 @@ async def call(self, spec: CallSpec) -> tuple[Any, dict[str, Any]]: raise EvaluationError( f"profile {spec.profile.name} is marked unavailable: {spec.profile.note}" ) + kwargs = _request_kwargs(spec) index = self.ledger.reserve(spec) started = time.monotonic() try: - response = await self.client.chat.completions.create(**_request_kwargs(spec)) + response = await self.client.chat.completions.create(**kwargs) except Exception as exc: latency = round((time.monotonic() - started) * 1000) self.ledger.finish( @@ -669,6 +741,18 @@ async def call(self, spec: CallSpec) -> tuple[Any, dict[str, Any]]: visible = _visible_response(response) usage = _usage(response) status = _response_status(visible, spec.profile) + cache_counts = [visible["usage_details"][key] for key in ( + "cached_tokens", "cache_write_tokens" + )] + visible["prompt_cache_observation"] = ( + "activity_reported" if any(count is not None and count > 0 for count in cache_counts) + else "none_reported" if cache_counts == [0, 0] else "unknown" + ) + if ( + status == "completed" and spec.profile.prompt_cache == "disabled" + and visible["prompt_cache_observation"] == "activity_reported" + ): + status = "prompt_cache_policy_mismatch" if usage is None and status != "malformed_response": status = "missing_usage_or_cost" elif usage is not None and ( @@ -684,7 +768,9 @@ async def call(self, spec: CallSpec) -> tuple[Any, dict[str, Any]]: artifact = { "request": _request_record(spec), "response": visible, - "accounting": usage and {key: str(value) for key, value in usage.items()}, + "accounting": usage and { + key: str(value) if value is not None else None for key, value in usage.items() + }, "status": self.ledger.data["calls"][index]["status"], } destination = self.responses / f"{index + 1:03d}-{spec.request_id}.json" diff --git a/tests/test_story_eval.py b/tests/test_story_eval.py index 6db01f4..b3fb4c5 100644 --- a/tests/test_story_eval.py +++ b/tests/test_story_eval.py @@ -1,4 +1,5 @@ import copy +import dataclasses import json from decimal import Decimal from types import SimpleNamespace @@ -181,8 +182,8 @@ def forbidden(*args, **kwargs): assert plan["credentials_required"] is False assert plan["writer_screen"]["calls"] == 48 assert plan["reviewer_calibration"]["calls"] == 48 - assert plan["writer_screen"]["cost_upper_bound_usd"] == "20.40" - assert plan["reviewer_calibration"]["cost_upper_bound_usd"] == "7.20" + assert plan["writer_screen"]["cost_upper_bound_usd"] == "21.84" + assert plan["reviewer_calibration"]["cost_upper_bound_usd"] == "8.16" assert plan["selected_pair_workflow"]["maximum_calls_for_two_pairings"] == 96 @@ -200,17 +201,167 @@ def test_input_reservation_counts_utf8_bytes_and_full_schema(): assert spec.inventory()["input_token_reserve"] == 32768 +def test_openai_reservations_cover_possible_prompt_cache_writes(): + _, cases, reviews = _fixture_data() + profiles = story_eval.load_profiles() + writer = story_eval._writer_spec(cases["straight_release"], profiles["sol"], 1) + reviewer = story_eval._reviewer_spec(reviews["good_release"], cases, profiles["astra"], 1) + + assert writer.reserved_cost_usd == Decimal("0.25") + assert reviewer.reserved_cost_usd == Decimal("0.82") + + +@pytest.mark.parametrize("price", [None, True, -1, "NaN", "Infinity", "unknown"]) +def test_invalid_cache_write_price_rejects_profile(price): + record = copy.deepcopy(story_eval._load_json(story_eval.DEFAULT_MODELS)["profiles"]["sol"]) + record["pricing_usd_per_million"]["prompt_cache_write"] = price + with pytest.raises(story_eval.EvaluationError, match="profile"): + story_eval.ModelProfile.from_record("sol", record) + + +@pytest.mark.parametrize("field", ["prompt_cache", "prompt_cache_write"]) +def test_missing_cache_policy_or_price_rejects_profile(field): + record = copy.deepcopy(story_eval._load_json(story_eval.DEFAULT_MODELS)["profiles"]["sol"]) + del (record if field == "prompt_cache" else record["pricing_usd_per_million"])[field] + with pytest.raises(story_eval.EvaluationError, match="prompt-cache profile"): + story_eval.ModelProfile.from_record("sol", record) + + +def test_unsupported_cache_off_policy_rejects_profile(): + record = copy.deepcopy(story_eval._load_json(story_eval.DEFAULT_MODELS)["profiles"]["sonnet"]) + record["prompt_cache"] = "disabled" + with pytest.raises(story_eval.EvaluationError, match="invalid model profile"): + story_eval.ModelProfile.from_record("sonnet", record) + + @pytest.mark.asyncio -async def test_sdk_request_is_exactly_pinned_and_has_no_retry_or_sampling(tmp_path): +async def test_cache_write_reservation_rejects_base_price_cap_before_io(tmp_path): _, cases, _ = _fixture_data() - profile = story_eval.load_profiles()["sol"] + spec = story_eval._writer_spec(cases["straight_release"], story_eval.load_profiles()["sol"], 1) + calls = 0 + + async def create(**kwargs): + nonlocal calls + calls += 1 + raise AssertionError("provider called before full reservation") + + client = SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=create))) + ledger = story_eval.Ledger(tmp_path / "ledger.json", max_calls=1, max_cost_usd=Decimal("0.23")) + with pytest.raises(story_eval.EvaluationError, match="cost budget"): + await story_eval.Runner(client, ledger, tmp_path / "responses").call(spec) + assert calls == 0 + assert ledger.data["reserved_calls"] == 0 + + +@pytest.mark.parametrize("message", [ + {"role": "user", "content": [{"type": "text", "text": "example", + "prompt_cache_breakpoint": {"ttl": "12h"}}]}, + {"role": "user", "content": "example", "cache_control": {"type": "ephemeral"}}, +]) +def test_unpriced_explicit_message_cache_controls_are_rejected(message): + _, cases, _ = _fixture_data() + spec = story_eval._writer_spec(cases["straight_release"], story_eval.load_profiles()["sol"], 1) + with pytest.raises(story_eval.EvaluationError, match="without cache controls"): + story_eval._request_kwargs(dataclasses.replace(spec, messages=[message])) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy,invalid_usage,expected", [ + ("provider_default", False, "completed"), + ("disabled", False, "prompt_cache_policy_mismatch"), + ("disabled", True, "missing_usage_or_cost"), + ("disabled", "negative_prompt_tokens", "missing_usage_or_cost"), +]) +async def test_cache_write_evidence_preserves_reported_cost_and_missing_usage( + tmp_path, policy, invalid_usage, expected +): + _, cases, _ = _fixture_data() + profile = dataclasses.replace(story_eval.load_profiles()["sol"], prompt_cache=policy) spec = story_eval._writer_spec(cases["straight_release"], profile, 1) + payload = _completion("{}", cost=0.006) + payload["id"] = "gen-synthetic-cache-cost" + payload["usage"].update({ + "prompt_tokens": 1600, "completion_tokens": 200, "total_tokens": 1800, + "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 1600, + "unexpected": "do not retain"}, + "cost_details": {"upstream_inference_cost": 0.006, + "upstream_inference_prompt_cost": 0.004, + "upstream_inference_completions_cost": 0.002, + "unexpected": "do not retain"}, + }) + if invalid_usage == "negative_prompt_tokens": + payload["usage"]["prompt_tokens"] = -1 + elif invalid_usage: + del payload["usage"]["cost"] + + async def create(**kwargs): + return ChatCompletion.model_validate(payload) + + client = SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=create))) + ledger = story_eval.Ledger(tmp_path / "ledger.json", max_calls=1, max_cost_usd=Decimal("1")) + responses = tmp_path / "responses" + responses.mkdir() + runner = story_eval.Runner(client, ledger, responses) + if expected == "completed": + await runner.call(spec) + else: + with pytest.raises(story_eval.EvaluationError, match=expected): + await runner.call(spec) + artifact = json.loads(next(responses.iterdir()).read_text()) + assert artifact["response"]["generation_id"] == "gen-synthetic-cache-cost" + assert artifact["response"]["usage_details"] == { + "cached_tokens": 0, "cache_write_tokens": 1600, + "cost_details": {"upstream_inference_cost": "0.006", + "upstream_inference_prompt_cost": "0.004", + "upstream_inference_completions_cost": "0.002"}, + } + assert artifact["response"]["prompt_cache_observation"] == "activity_reported" + assert "do not retain" not in json.dumps(artifact) + assert ledger.data["used_cost_usd"] == ("0" if invalid_usage else "0.006") + assert ledger.data["reserved_cost_usd"] == "0.25" + assert artifact["status"] == expected + + +@pytest.mark.parametrize("invalid", [True, -1, "NaN", "Infinity", "private text", {}]) +def test_optional_usage_metadata_rejects_invalid_values_without_losing_cost(invalid): + payload = _completion("{}") + payload["id"] = "not a generation identifier\nprivate text" + response = ChatCompletion.model_construct( + **{**payload, "usage": SimpleNamespace( + cost=0.01, prompt_tokens=100, completion_tokens=25, + completion_tokens_details=None, + prompt_tokens_details={"cached_tokens": invalid, "cache_write_tokens": invalid}, + cost_details={"upstream_inference_cost": invalid}, + )} + ) + visible = story_eval._visible_response(response) + assert visible["generation_id"] is None + assert visible["usage_details"] == { + "cached_tokens": None, "cache_write_tokens": None, + "cost_details": {"upstream_inference_cost": None, + "upstream_inference_prompt_cost": None, + "upstream_inference_completions_cost": None}, + } + assert story_eval._usage(response)["cost_usd"] == Decimal("0.01") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("profile_name", ["sol", "astra"]) +async def test_sdk_request_is_exactly_pinned_and_has_no_retry_or_sampling(tmp_path, profile_name): + _, cases, reviews = _fixture_data() + profile = story_eval.load_profiles()[profile_name] + spec = ( + story_eval._writer_spec(cases["straight_release"], profile, 1) if profile_name == "sol" + else story_eval._reviewer_spec(reviews["good_release"], cases, profile, 1) + ) requests = [] async def handle(request): requests.append(request) content = json.dumps(cases["straight_release"]["reference_decision"]) - return httpx.Response(200, json=_completion(content)) + payload = _completion(content, model=profile.returned_models[-1]) + payload["openrouter_metadata"]["requested"] = profile.model + return httpx.Response(200, json=payload) http_client = httpx.AsyncClient(transport=httpx.MockTransport(handle)) client = AsyncOpenAI( @@ -227,34 +378,45 @@ async def handle(request): assert len(requests) == 1 request = json.loads(requests[0].content) - assert request["model"] == "openai/gpt-6.1-sol-20260929" + assert request["model"] == { + "sol": "openai/gpt-6.1-sol-20260929", "astra": "openai/gpt-6-astra-20260903" + }[profile_name] assert request["service_tier"] == "default" - assert request["max_tokens"] == 16384 + assert request["max_tokens"] == (16384 if profile_name == "sol" else 8192) assert request["provider"] == { "only": ["openai"], "allow_fallbacks": False, "require_parameters": True, - "max_price": {"prompt": 2.0, "completion": 10.0}, + "max_price": {"prompt": 2.0, "completion": 10.0} if profile_name == "sol" else { + "prompt": 10.0, "completion": 50.0 + }, } assert request["reasoning"] == {"effort": "medium", "exclude": True} + assert request["prompt_cache_options"] == {"mode": "explicit"} + assert all(set(message) == {"role", "content"} for message in request["messages"]) + assert all(isinstance(message["content"], str) for message in request["messages"]) assert request["response_format"]["json_schema"]["strict"] is True assert not {"temperature", "top_p", "models"}.intersection(request) assert requests[0].headers["x-openrouter-metadata"] == "enabled" assert requests[0].headers["x-openrouter-cache"] == "false" + artifact = json.loads(next(responses.iterdir()).read_text()) + assert artifact["request"]["response_cache"] == "disabled" + assert artifact["request"]["prompt_cache"] == "disabled" + assert artifact["response"]["prompt_cache_observation"] == "unknown" def test_ledger_reserves_before_calls_and_enforces_both_limits(tmp_path): _, cases, _ = _fixture_data() spec = story_eval._writer_spec(cases["straight_release"], story_eval.load_profiles()["sol"], 1) - ledger = story_eval.Ledger(tmp_path / "ledger.json", max_calls=1, max_cost_usd=Decimal("0.23")) + ledger = story_eval.Ledger(tmp_path / "ledger.json", max_calls=1, max_cost_usd=Decimal("0.25")) ledger.reserve(spec) durable = json.loads((tmp_path / "ledger.json").read_text()) assert durable["reserved_calls"] == 1 - assert durable["reserved_cost_usd"] == "0.23" + assert durable["reserved_cost_usd"] == "0.25" with pytest.raises(story_eval.EvaluationError, match="call budget"): ledger.reserve(spec) - costly = story_eval.Ledger(tmp_path / "cost.json", max_calls=2, max_cost_usd=Decimal("0.22")) + costly = story_eval.Ledger(tmp_path / "cost.json", max_calls=2, max_cost_usd=Decimal("0.24")) with pytest.raises(story_eval.EvaluationError, match="cost budget"): costly.reserve(spec) @@ -382,6 +544,11 @@ async def handle(request): await client.close() assert ledger.data["calls"][0]["status"] == "completed" assert ledger.data["calls"][0]["reasoning_tokens"] is None + artifact = json.loads(next(responses.iterdir()).read_text()) + assert artifact["accounting"]["reasoning_tokens"] is None + assert artifact["response"]["usage_details"]["cached_tokens"] is None + assert artifact["response"]["usage_details"]["cache_write_tokens"] is None + assert artifact["response"]["prompt_cache_observation"] == "unknown" @pytest.mark.asyncio @@ -407,9 +574,11 @@ async def test_malformed_sdk_response_persists_safe_artifacts_and_accounting( completion_tokens=25, cost=0.01, completion_tokens_details=SimpleNamespace(reasoning_tokens=11), + prompt_tokens_details=SimpleNamespace(cached_tokens=0, cache_write_tokens=10), + cost_details={"upstream_inference_cost": 0.01}, ) response = ChatCompletion.model_construct( - id="eval-response", + id="gen-malformed-response", object="chat.completion", created=1, model="openai/gpt-6.1-sol", @@ -443,7 +612,7 @@ async def create(**kwargs): assert row["status"] == "malformed_response" assert durable["reserved_calls"] == 1 assert durable["used_calls"] == (1 if include_usage else 0) - assert durable["reserved_cost_usd"] == "0.23" + assert durable["reserved_cost_usd"] == "0.25" assert durable["used_cost_usd"] == ("0.01" if include_usage else "0") artifact = json.loads(next(responses.iterdir()).read_text()) @@ -451,6 +620,10 @@ async def create(**kwargs): assert artifact["response"]["shape_error"] == shape_error assert artifact["response"]["model"] == "openai/gpt-6.1-sol" assert artifact["response"]["route_metadata"]["attempt"] == 1 + assert artifact["response"]["generation_id"] == "gen-malformed-response" + assert artifact["response"]["usage_details"]["cache_write_tokens"] == ( + 10 if include_usage else None + ) assert (artifact["accounting"] is not None) is include_usage assert "choices" not in artifact["response"] assert "reasoning" not in artifact["response"]