diff --git a/specs/img/860-codex-cost-before-after.png b/specs/img/860-codex-cost-before-after.png new file mode 100644 index 00000000..f5b70987 Binary files /dev/null and b/specs/img/860-codex-cost-before-after.png differ diff --git a/src/pinky_daemon/agent_registry.py b/src/pinky_daemon/agent_registry.py index 22cf1d28..c5a09442 100644 --- a/src/pinky_daemon/agent_registry.py +++ b/src/pinky_daemon/agent_registry.py @@ -3966,7 +3966,8 @@ def delete_bot_token(self, token_id: str) -> bool: ("anthropic", "claude-opus-4-5", "Claude Opus 4.5", "Previous-gen Opus.", "opus", 200_000, 0, 5.0, 25.0, 0.5, 1, 40), ("anthropic", "claude-sonnet-4-5", "Claude Sonnet 4.5", "Previous-gen Sonnet.", "sonnet", 200_000, 0, 3.0, 15.0, 0.3, 1, 50), # OpenAI / Codex CLI - ("openai", "gpt-5.5", "GPT-5.5", "Newest frontier. Coding + reasoning. Codex sign-in auth only (API pending).", "flagship", 200_000, 0, 1.75, 14.0, 0.175, 0, 55), + ("openai", "gpt-5.6-sol", "GPT-5.6 Sol", "Current codex fleet model (2026-07). Frontier coding + reasoning. Codex sign-in auth only (API pending).", "flagship", 200_000, 0, 5.0, 30.0, 0.5, 0, 54), + ("openai", "gpt-5.5", "GPT-5.5", "Previous frontier. Coding + reasoning. Codex sign-in auth only (API pending).", "flagship", 200_000, 0, 5.0, 30.0, 0.5, 0, 55), ("openai", "gpt-5.4", "GPT-5.4", "Flagship. Complex reasoning & coding.", "flagship", 200_000, 0, 1.75, 14.0, 0.175, 0, 60), ("openai", "gpt-5.4-mini", "GPT-5.4 Mini", "Fast + capable. Daily driver.", "mid", 200_000, 0, 0.25, 2.0, 0.025, 0, 70), ("openai", "gpt-5.4-nano", "GPT-5.4 Nano", "Cheapest. High-volume tasks.", "low", 200_000, 0, 0.05, 0.4, 0.005, 0, 80), @@ -3986,6 +3987,10 @@ def delete_bot_token(self, token_id: str) -> bool: ("anthropic/claude-opus-4-6", (15.0, 75.0, 1.5), (5.0, 25.0, 0.5)), ("anthropic/claude-opus-4-5", (15.0, 75.0, 1.5), (5.0, 25.0, 0.5)), ("anthropic/claude-haiku-4-5", (0.8, 4.0, 0.08), (1.0, 5.0, 0.1)), + # #860: gpt-5.5 was seeded at the gpt-5.2-tier $1.75/$14 while the + # official rate is $5/$30 (cached $0.50) — analytics_store and + # pricing.RATE_TABLE always had it right; this realigns the catalog. + ("openai/gpt-5.5", (1.75, 14.0, 0.175), (5.0, 30.0, 0.5)), ] def _seed_models(self) -> None: diff --git a/src/pinky_daemon/analytics_store.py b/src/pinky_daemon/analytics_store.py index 576f64a2..4faac696 100644 --- a/src/pinky_daemon/analytics_store.py +++ b/src/pinky_daemon/analytics_store.py @@ -221,6 +221,7 @@ def _init_db(self) -> None: self._migrate_opus_haiku_seed_pricing(conn) self._migrate_legacy_dotted_model_ids(conn) self._ensure_pricing_rows(conn) + self._migrate_codex_provider_attribution(conn) # Schema migration: add user_message_snippet to turn_usage cols = { r[1] for r in conn.execute("PRAGMA table_info(analytics_turn_usage)").fetchall() @@ -258,6 +259,11 @@ def _init_db(self) -> None: _LATEST_PRICING_ADDITIONS = [ ("openai", "gpt-5.5", "2020-01-01T00:00:00Z", None, 5.00, 30.00, 0.50, "seed"), ("openai", "gpt-5.3-codex", "2020-01-01T00:00:00Z", None, 1.75, 14.00, 0.175, "seed"), + # gpt-5.6-sol (#860): codex fleet model since 2026-07-09. Official + # rates identical to gpt-5.5 (developers.openai.com, verified + # 2026-07-10). Mirrors pricing.RATE_TABLE (_GPT_FRONTIER), parity-pinned + # by test_seed_pricing_matches_rate_table. + ("openai", "gpt-5.6-sol", "2020-01-01T00:00:00Z", None, 5.00, 30.00, 0.50, "seed"), # Sonnet 5 (2026-06): the current Sonnet tier. Standard $3/$15 (cached # $0.30) — mirrors pinky_daemon.pricing.RATE_TABLE (_SONNET), pinned by # test_seed_pricing_matches_rate_table. Introductory $2/$10 runs through @@ -470,6 +476,55 @@ def _ensure_pricing_rows(self, conn) -> None: (*r, r[0], r[1]), ) + def _migrate_codex_provider_attribution(self, conn) -> None: + """Reattribute + reprice codex turns mislogged as ``anthropic`` (#860). + + ``TmuxSession._log_turn_cost_and_analytics`` hardcoded + ``provider="anthropic"``, so every CodexTmuxSession turn landed as + ``anthropic/gpt-*`` — a (provider, model) pair the pricing lookup can + never match (``_provider_alias`` never crosses providers), pricing all + codex history at $0. Heal deployed rows to ``codex_cli`` — the value + new codex turns record and the SDK path's default — which aliases onto + the openai rate rows at read time. + + The provider flip alone would over-bill history: pre-#860 rows also + predate ``_normalize_turn_usage``, so their ``input_tokens`` is + codex's INCLUSIVE count with the cached span invisible + (``cached_input_tokens=0`` — the old extraction read only the + Anthropic cache keys). Repricing that at the full input rate + overstates ~7x (measured across the fleet's 2026-07 rollouts: 97.5% + of codex input is cached reads). The per-turn split is unrecoverable + from the DB, so rows carrying that signature move the whole count to + the cached column: error bounded at roughly -20% (the true-uncached + share priced at 10x the cached rate), failing toward the visible + undercount — the same direction ``_normalize_turn_usage`` chooses. + + Follows the ``_migrate_opus_haiku_seed_pricing`` precedent: idempotent + (both predicates match only misattributed ``anthropic`` pairs, gone + after the first run) and narrow (a ``gpt-*`` model under ``anthropic`` + cannot be legitimate — Anthropic serves no gpt model — so no honest + row can match). Touches turn usage AND session facts so per-session + rollups group consistently. + """ + conn.execute( + """ + UPDATE analytics_turn_usage + SET provider = 'codex_cli', + cached_input_tokens = input_tokens, + input_tokens = 0 + WHERE provider = 'anthropic' AND model LIKE 'gpt-%' + AND cached_input_tokens = 0 + """ + ) + for table in ("analytics_turn_usage", "analytics_session_facts"): + conn.execute( + f""" + UPDATE {table} + SET provider = 'codex_cli' + WHERE provider = 'anthropic' AND model LIKE 'gpt-%' + """ + ) + def ensure_session_fact( self, *, diff --git a/src/pinky_daemon/codex_tmux_session.py b/src/pinky_daemon/codex_tmux_session.py index 38f7a83d..db8ba471 100644 --- a/src/pinky_daemon/codex_tmux_session.py +++ b/src/pinky_daemon/codex_tmux_session.py @@ -86,6 +86,43 @@ async def paste_text( class CodexTmuxSession(TmuxSession): """Interactive codex-CLI REPL in a detached tmux pane (see module docstring).""" + # #860: analytics rows must not claim "anthropic" for codex turns — the + # pricing lookup keys on (provider, model) and never crosses providers, so + # anthropic/gpt-* priced every turn at $0. "codex_cli" matches the SDK + # path's default (CodexSession._analytics_log_turn_usage) and + # analytics_store._provider_alias maps it onto the openai rate rows. + _ANALYTICS_PROVIDER = "codex_cli" + + @staticmethod + def _normalize_turn_usage(u: dict) -> dict: + """Codex usage schema → the daemon's disjoint convention (#860). + + Codex reports ``input_tokens`` INCLUSIVE of the cached prefix, with the + cached span under ``cached_input_tokens`` — while every consumer here + (SessionUsage accumulation, ``compute_cost_from_usage``, analytics + ``log_turn_usage``, the context gauge's window reconstruction) follows + the Anthropic convention: ``input_tokens`` is the uncached remainder + and the cached span rides ``cache_read_input_tokens``. Mirrors + ``CodexSession.uncached_input_tokens`` on the SDK path; pricing the + inclusive number at the full input rate would overstate review-heavy + sessions roughly 10x. The ambiguous source key is consumed, not + forwarded. A dict without ``cached_input_tokens`` (or with malformed + counts) passes through untouched — fail toward the visible undercount, + never a silent double-bill. + """ + if "cached_input_tokens" not in u: + return u + try: + cached = max(0, int(u.get("cached_input_tokens") or 0)) + inclusive = max(0, int(u.get("input_tokens") or 0)) + except (TypeError, ValueError): + return u + out = dict(u) + out.pop("cached_input_tokens", None) + out["input_tokens"] = max(0, inclusive - cached) + out["cache_read_input_tokens"] = cached + return out + def __init__( self, config: StreamingSessionConfig, diff --git a/src/pinky_daemon/pricing.py b/src/pinky_daemon/pricing.py index 0582912f..8d85bd4e 100644 --- a/src/pinky_daemon/pricing.py +++ b/src/pinky_daemon/pricing.py @@ -93,6 +93,39 @@ "cache_write_5m": 1.00, "cache_write_1h": 1.60, } +# OpenAI / Codex family (#860): powers the live tmux cost path for codex +# agents (CodexTmuxSession rides the same _log_turn_cost_and_analytics as +# Claude tmux). Official OpenAI API rates (developers.openai.com, verified +# 2026-07-10); codex agents run on sign-in subscription, so as with CC +# subscription agents these are notional usage-value figures (#648). +# Cache-write billing DIFFERS per model page: gpt-5.6-sol documents cache +# writes at 1.25x the uncached input rate ($6.25/Mtok); the gpt-5.5 and +# gpt-5.3-codex pages list no write tariff, so those bill $0. OpenAI has no +# 5m/1h TTL split — the documented tariff goes in both positions. Cached +# reads use the published cached-input rate. Must mirror the analytics +# openai seeds (test_seed_pricing_matches_rate_table, #669 — that guard +# pins input/output/cache_read; the seed table has no write column). +_GPT_56_SOL = { + "input": 5.00, + "output": 30.00, + "cache_read": 0.50, + "cache_write_5m": 6.25, # 1.25x input per the sol model page + "cache_write_1h": 6.25, +} +_GPT_55 = { + "input": 5.00, + "output": 30.00, + "cache_read": 0.50, + "cache_write_5m": 0.0, + "cache_write_1h": 0.0, +} +_GPT_53_CODEX = { + "input": 1.75, + "output": 14.00, + "cache_read": 0.175, + "cache_write_5m": 0.0, + "cache_write_1h": 0.0, +} # Bare-model-id → rate dict. Add new model ids here on each release. RATE_TABLE: dict[str, dict[str, float]] = { @@ -117,6 +150,12 @@ "claude-sonnet-4": _SONNET, "claude-haiku-4-5": _HAIKU_45, "claude-haiku-3-5": _HAIKU_35, + # Codex fleet models (#860). gpt-5.6-sol is the current fleet model + # (2026-07-09→); gpt-5.5 the previous one; gpt-5.3-codex kept because the + # analytics seed carries it (parity-pinned). + "gpt-5.6-sol": _GPT_56_SOL, + "gpt-5.5": _GPT_55, + "gpt-5.3-codex": _GPT_53_CODEX, } diff --git a/src/pinky_daemon/tmux_session.py b/src/pinky_daemon/tmux_session.py index 0131fe3c..853fd2a5 100644 --- a/src/pinky_daemon/tmux_session.py +++ b/src/pinky_daemon/tmux_session.py @@ -3804,6 +3804,27 @@ def _soft_nudge_threshold_pct(self) -> float: pass return DEFAULT_CONTEXT_NUDGE_THRESHOLD_PCT + # Provider recorded on analytics_turn_usage rows (#860). Class-level so + # transport subclasses that ride this same cost/analytics path attribute + # their turns truthfully — CodexTmuxSession overrides with "codex_cli" + # (analytics_store._provider_alias maps it to the openai rate rows). + _ANALYTICS_PROVIDER: str = "anthropic" + + @staticmethod + def _normalize_turn_usage(u: dict) -> dict: + """Transport-specific usage normalization hook (#860). + + The base (Claude) transport already emits the daemon's disjoint + convention, so this is the identity. CodexTmuxSession overrides it to + convert the codex schema (``input_tokens`` INCLUSIVE of the cached + prefix, cached span under ``cached_input_tokens``) before any + consumer — usage accumulation, pricing, analytics, context gauge — + reads the dict. ONE conversion point by design: a partial conversion + spread across consumers is how the cached split got silently dropped + in the first place. + """ + return u + def _record_turn_usage(self, response: TurnResponse) -> None: """Fold a turn's usage block into ``self.usage`` (SessionUsage). @@ -3929,7 +3950,10 @@ def _log_turn_cost_and_analytics(self, response: TurnResponse) -> None: session_id=self.id, agent_name=self.agent_name, turn_seq=turn_seq, - provider="anthropic", + # #860: was hardcoded "anthropic", which mispriced every + # CodexTmuxSession turn to $0 (anthropic/gpt-* never + # matches the openai rate rows). + provider=self._ANALYTICS_PROVIDER, model=model, input_tokens=input_tokens, output_tokens=output_tokens, @@ -4380,6 +4404,10 @@ async def _handle_turn_complete(self, response: TurnResponse) -> None: # tailer already pulled the usage dict out of each assistant # entry's ``usage`` block; we just need to fold it into the # session-level dataclass and emit it. + # #860: normalize transport-specific usage schemas ONCE, before any + # consumer (accumulation, pricing, analytics, context gauge) reads it. + if isinstance(response.usage, dict): + response.usage = self._normalize_turn_usage(response.usage) self._record_turn_usage(response) # #648 — forward per-turn usage to analytics + cost tracking so # tmux agents reach live Analytics / lifetime-cost parity with the diff --git a/tests/test_agent_registry.py b/tests/test_agent_registry.py index 6e567e93..3fc5e85e 100644 --- a/tests/test_agent_registry.py +++ b/tests/test_agent_registry.py @@ -995,3 +995,65 @@ def test_operator_customized_prices_survive_reseed(self, registry): if m["id"] == "anthropic/claude-opus-4-8" ) assert opus["input_price"] == 12.34 + + def test_gpt_56_sol_seeded_at_frontier_rates(self, registry): + """#860: gpt-5.6-sol (the codex fleet model since 2026-07) must be in + the catalog at the official $5/$30 (cached $0.50).""" + models = { + m["model_id"]: m + for m in registry.list_models(provider="openai", active_only=False) + } + assert "gpt-5.6-sol" in models + sol = models["gpt-5.6-sol"] + assert (sol["input_price"], sol["output_price"], + sol["cached_input_price"]) == (5.0, 30.0, 0.5) + assert sol["tier"] == "flagship" + + def test_openai_seed_prices_match_pricing_rate_table(self, registry): + """#860 extends the #741 invariant to the OpenAI family: catalog + display prices must agree with pricing.py (the actual cost engine) + for every openai model both tables know.""" + from pinky_daemon.pricing import RATE_TABLE + + checked = 0 + for m in registry.list_models(provider="openai", active_only=False): + rate = RATE_TABLE.get(m["model_id"]) + if rate is None: + continue + assert m["input_price"] == rate["input"], m["model_id"] + assert m["output_price"] == rate["output"], m["model_id"] + assert m["cached_input_price"] == rate["cache_read"], m["model_id"] + checked += 1 + # Guard the guard: gpt-5.6-sol + gpt-5.5 must both be intersecting. + assert checked >= 2 + + def test_stale_gpt55_price_corrected_on_existing_rows(self, registry): + """#860: deployed DBs seeded gpt-5.5 at the gpt-5.2-tier $1.75/$14; + the next seed pass realigns existing rows to the official $5/$30 + (INSERT OR IGNORE alone never reaches them).""" + registry._db.execute( + "UPDATE models SET input_price=1.75, output_price=14.0," + " cached_input_price=0.175 WHERE id='openai/gpt-5.5'" + ) + registry._db.commit() + registry._seed_models() + gpt = next( + m for m in registry.list_models(provider="openai", active_only=False) + if m["id"] == "openai/gpt-5.5" + ) + assert (gpt["input_price"], gpt["output_price"], + gpt["cached_input_price"]) == (5.0, 30.0, 0.5) + + def test_operator_customized_gpt55_price_survives_reseed(self, registry): + """Same exact-stale-triple gate as the Anthropic corrections — an + operator-priced gpt-5.5 row must not be clobbered.""" + registry._db.execute( + "UPDATE models SET input_price=9.99 WHERE id='openai/gpt-5.5'" + ) + registry._db.commit() + registry._seed_models() + gpt = next( + m for m in registry.list_models(provider="openai", active_only=False) + if m["id"] == "openai/gpt-5.5" + ) + assert gpt["input_price"] == 9.99 diff --git a/tests/test_analytics_store.py b/tests/test_analytics_store.py index 17c578ca..82fbc3b0 100644 --- a/tests/test_analytics_store.py +++ b/tests/test_analytics_store.py @@ -456,25 +456,38 @@ class TestSeedRateTableParity: per-turn). They must agree for the same model, or the same usage reports different dollar figures. This pins them together as a drift guard.""" + # RATE_TABLE keys are bare model ids; the Analytics seed keys on + # (provider, model). #860 added the OpenAI/codex family to RATE_TABLE, so + # the guard is provider-aware: each family must be seeded under the RIGHT + # provider (a gpt row seeded as anthropic would be unpriceable at read + # time — _provider_alias never crosses providers). + @staticmethod + def _expected_provider(model: str) -> str: + return "anthropic" if model.startswith("claude-") else "openai" + def test_seed_pricing_matches_rate_table(self, tmp_path): store = _store(tmp_path) with store._connect() as conn: rows = conn.execute( - "SELECT model, input_usd_per_mtok, output_usd_per_mtok, " - "cached_input_usd_per_mtok FROM analytics_model_pricing " - "WHERE provider = 'anthropic'" + "SELECT provider, model, input_usd_per_mtok, output_usd_per_mtok, " + "cached_input_usd_per_mtok FROM analytics_model_pricing" ).fetchall() # Seed ids are canonical (hyphenated) since #759, so they line up with # RATE_TABLE directly — no normalization. A dotted-id regression would # drop the matching RATE_TABLE key out of `seeded` and fail the check. - seeded = {r["model"]: r for r in rows} + seeded = {(r["provider"], r["model"]): r for r in rows} - missing = [m for m in RATE_TABLE if m not in seeded] - assert not missing, f"models in RATE_TABLE but absent from Analytics seed: {missing}" + missing = [ + m for m in RATE_TABLE if (self._expected_provider(m), m) not in seeded + ] + assert not missing, ( + f"models in RATE_TABLE but absent from the Analytics seed " + f"(under their expected provider): {missing}" + ) mismatches = [] for model, rates in RATE_TABLE.items(): - row = seeded[model] + row = seeded[(self._expected_provider(model), model)] if ( row["input_usd_per_mtok"] != rates["input"] or row["output_usd_per_mtok"] != rates["output"] @@ -588,6 +601,130 @@ def test_migration_preserves_operator_overrides(self, tmp_path): assert row is not None and row[0] == 15.00 +class TestCodexProviderAttributionMigration: + """#860: TmuxSession hardcoded provider="anthropic" at the log_turn_usage + call, so every CodexTmuxSession turn landed as anthropic/gpt-* — a + (provider, model) pair the pricing lookup can never match, costing all + codex tmux history at $0. _migrate_codex_provider_attribution heals + deployed rows to codex_cli on init (turn usage AND session facts), and + moves the pre-normalization INCLUSIVE input count to the cached column + (pricing it at the full input rate would overstate ~7x; see migration + docstring).""" + + def _log_mislogged_codex_turn(self, store: AnalyticsStore) -> None: + """One turn exactly as the pre-#860 tmux path wrote it: inclusive + input count, cached span invisible (cached_input_tokens=0).""" + store.ensure_session_fact( + session_id="codex1", agent_name="murzik", session_label="test", + provider="anthropic", model="gpt-5.5", + ) + store.log_turn_usage( + session_id="codex1", agent_name="murzik", turn_seq=1, + provider="anthropic", model="gpt-5.5", + input_tokens=1_000_000, output_tokens=0, cached_input_tokens=0, + ) + + def test_heals_both_tables_and_restores_pricing(self, tmp_path): + db = str(tmp_path / "analytics.db") + store = AnalyticsStore(db) + self._log_mislogged_codex_turn(store) + # The bug as deployed: anthropic/gpt-5.5 matches no rate row → $0. + overview = store.get_overview(range_name="7d") + assert overview["totals"]["cost_usd"] == pytest.approx(0.0) + + store2 = AnalyticsStore(db) # init runs the migration + with store2._connect() as conn: + tu = conn.execute( + "SELECT provider, input_tokens, cached_input_tokens " + "FROM analytics_turn_usage WHERE session_id='codex1'" + ).fetchall() + sf = conn.execute( + "SELECT provider FROM analytics_session_facts WHERE session_id='codex1'" + ).fetchall() + assert [r["provider"] for r in tu] == ["codex_cli"] + assert [r["provider"] for r in sf] == ["codex_cli"] + # The inclusive input count moved to the cached column. + assert tu[0]["input_tokens"] == 0 + assert tu[0]["cached_input_tokens"] == 1_000_000 + # History is repriced, not just relabeled: 1M cached @ $0.50/Mtok. + overview = store2.get_overview(range_name="7d") + assert overview["totals"]["cost_usd"] == pytest.approx(0.5) + + def test_row_with_cached_split_gets_provider_flip_only(self, tmp_path): + """A mislogged row that somehow carries a real cached split doesn't + match the pre-normalization signature — its tokens are already + disjoint, so only the provider is healed.""" + db = str(tmp_path / "analytics.db") + store = AnalyticsStore(db) + store.ensure_session_fact( + session_id="codex2", agent_name="murzik", session_label="test", + provider="anthropic", model="gpt-5.5", + ) + store.log_turn_usage( + session_id="codex2", agent_name="murzik", turn_seq=1, + provider="anthropic", model="gpt-5.5", + input_tokens=10_000, output_tokens=0, cached_input_tokens=90_000, + ) + store2 = AnalyticsStore(db) + with store2._connect() as conn: + row = conn.execute( + "SELECT provider, input_tokens, cached_input_tokens " + "FROM analytics_turn_usage WHERE session_id='codex2'" + ).fetchone() + assert row["provider"] == "codex_cli" + assert row["input_tokens"] == 10_000 + assert row["cached_input_tokens"] == 90_000 + + def test_narrow_leaves_honest_rows_alone(self, tmp_path): + """A gpt-* model under anthropic cannot be legitimate, but every + honest (provider, model) combination must survive untouched.""" + db = str(tmp_path / "analytics.db") + store = AnalyticsStore(db) + _seed_session(store) + honest = [ + ("a1", "anthropic", "claude-opus-4-8"), + ("c1", "codex_cli", "gpt-5.5"), + ("o1", "openai", "gpt-5.5"), + ] + for i, (sid, provider, model) in enumerate(honest, start=1): + store.log_turn_usage( + session_id=sid, agent_name="barsik", turn_seq=i, + provider=provider, model=model, + input_tokens=100, output_tokens=10, cached_input_tokens=0, + ) + store2 = AnalyticsStore(db) + with store2._connect() as conn: + rows = conn.execute( + "SELECT session_id, provider FROM analytics_turn_usage" + ).fetchall() + providers = {r["session_id"]: r["provider"] for r in rows} + assert providers == {"a1": "anthropic", "c1": "codex_cli", "o1": "openai"} + # The seeded session fact (anthropic/claude-sonnet-4) is honest too. + with store2._connect() as conn: + fact = conn.execute( + "SELECT provider FROM analytics_session_facts WHERE session_id='sess1'" + ).fetchone() + assert fact["provider"] == "anthropic" + + def test_idempotent_on_reopen(self, tmp_path): + """Both heal predicates key on provider='anthropic', gone after the + first run — a second init must not re-shift tokens or duplicate.""" + db = str(tmp_path / "analytics.db") + store = AnalyticsStore(db) + self._log_mislogged_codex_turn(store) + AnalyticsStore(db) # heals + store3 = AnalyticsStore(db) # second run must be a no-op + with store3._connect() as conn: + rows = conn.execute( + "SELECT provider, model, input_tokens, cached_input_tokens " + "FROM analytics_turn_usage WHERE session_id='codex1'" + ).fetchall() + assert len(rows) == 1 + assert (rows[0]["provider"], rows[0]["model"]) == ("codex_cli", "gpt-5.5") + assert rows[0]["input_tokens"] == 0 + assert rows[0]["cached_input_tokens"] == 1_000_000 + + class TestLegacyDottedIdMigration: """#759: deployed DBs seeded before the canonicalization keep dotted legacy ids. _migrate_legacy_dotted_model_ids renames them to the hyphenated form on diff --git a/tests/test_codex_tmux_session.py b/tests/test_codex_tmux_session.py index 073d40d3..5993088b 100644 --- a/tests/test_codex_tmux_session.py +++ b/tests/test_codex_tmux_session.py @@ -13,6 +13,7 @@ """ import asyncio import os +import time from unittest.mock import AsyncMock, MagicMock import pytest @@ -24,7 +25,15 @@ ) from pinky_daemon.codex_tmux_transcript import CodexTmuxTranscriptTailer from pinky_daemon.streaming_session import StreamingSessionConfig -from pinky_daemon.tmux_session import TmuxCommandResult, _TmuxControl +from pinky_daemon.tmux_session import ( + TmuxCommandResult, + TmuxSession, + _InflightMeta, + _QueuedTurn, + _TmuxControl, +) +from pinky_daemon.tmux_transcript import TurnResponse +from pinky_daemon.transport_state import SessionState def _ok(stdout: str = "") -> TmuxCommandResult: @@ -371,6 +380,133 @@ def test_container_isolation_transport_checked_before_runtime(): assert status == 400 +# ── #860 — cost attribution + usage normalization ─────────────────────────── +# The codex transport shares TmuxSession._log_turn_cost_and_analytics; its two +# seams are _ANALYTICS_PROVIDER (truthful provider on analytics rows) and +# _normalize_turn_usage (codex's inclusive input_tokens + cached_input_tokens +# → the daemon's disjoint convention, once, before any consumer). + + +def test_analytics_provider_is_codex_cli(): + """Codex turns must not claim "anthropic": the pricing lookup keys on + (provider, model) and _provider_alias never crosses providers, so + anthropic/gpt-* rows price at $0. "codex_cli" matches the SDK path's + default and aliases onto the openai rate rows.""" + assert CodexTmuxSession._ANALYTICS_PROVIDER == "codex_cli" + assert TmuxSession._ANALYTICS_PROVIDER == "anthropic" + + +def test_normalize_converts_codex_schema_to_disjoint(): + """Codex reports input_tokens INCLUSIVE of the cached prefix; the daemon + convention is disjoint (input = uncached remainder, cached span under + cache_read_input_tokens). The ambiguous source key is consumed.""" + u = {"input_tokens": 100_000, "cached_input_tokens": 90_000, + "output_tokens": 500} + out = CodexTmuxSession._normalize_turn_usage(u) + assert out["input_tokens"] == 10_000 + assert out["cache_read_input_tokens"] == 90_000 + assert "cached_input_tokens" not in out + assert out["output_tokens"] == 500 + # The source dict is copied, never mutated in place. + assert u["input_tokens"] == 100_000 + assert u["cached_input_tokens"] == 90_000 + + +def test_normalize_without_cached_key_passes_through(): + """No cached_input_tokens ⇒ nothing to convert; the dict passes through + untouched (fail toward the visible undercount, never a double-bill).""" + u = {"input_tokens": 5_000, "output_tokens": 100} + assert CodexTmuxSession._normalize_turn_usage(u) is u + + +def test_normalize_malformed_counts_pass_through(): + u = {"input_tokens": 5_000, "cached_input_tokens": "garbage"} + assert CodexTmuxSession._normalize_turn_usage(u) is u + + +def test_normalize_clamps_cached_larger_than_inclusive(): + """cached > inclusive shouldn't happen, but must clamp to 0 rather than + log a negative input count.""" + u = {"input_tokens": 1_000, "cached_input_tokens": 2_000} + out = CodexTmuxSession._normalize_turn_usage(u) + assert out["input_tokens"] == 0 + assert out["cache_read_input_tokens"] == 2_000 + + +def _seed_inflight(ss: CodexTmuxSession) -> None: + """Minimal _InflightMeta seed so _handle_turn_complete has a head entry + (mirrors tests/test_tmux_session.py's helper).""" + ss._inflight_metas.append( + _InflightMeta( + meta={}, + completion_event=None, + internal=False, + dispatched_at=time.time(), + turn=_QueuedTurn(prompt=""), + ) + ) + ss._head_started_at = time.time() + + +@pytest.mark.asyncio +async def test_turn_complete_prices_codex_turn_correctly(): + """#860 end-to-end: a codex-schema turn (usage exactly as the tailer + snapshots token_count.info.last_token_usage) must log provider=codex_cli, + the DISJOINT token split, and the openai-rate dollar figure. Pre-#860 this + row logged as anthropic/gpt-* at $0 — and had a rate row existed, the + inclusive input count would have over-billed the cached span ~10x on + review-heavy sessions.""" + analytics = MagicMock() + cost_cb = MagicMock() + cfg = StreamingSessionConfig( + agent_name="murzik", + working_dir="/tmp/codex-tmux-test", + model="gpt-5.6-sol", + provider_key="sk-test", + ) + ss = CodexTmuxSession( + cfg, + tmux_control=_mock_tmux(), + analytics_store=analytics, + cost_callback=cost_cb, + ) + ss._skip_wake_prompt_for_tests = True + ss._state_machine._state = SessionState.CONNECTED + _seed_inflight(ss) + + await ss._handle_turn_complete( + TurnResponse( + text="ok", + stop_reason="end_turn", + model="gpt-5.6-sol", + usage={ + "input_tokens": 100_000, # inclusive of cached (codex) + "cached_input_tokens": 90_000, + "output_tokens": 1_000, + }, + duration_ms=100, + assistant_entry_count=1, + tool_uses=[], + ) + ) + + analytics.log_turn_usage.assert_called_once() + kwargs = analytics.log_turn_usage.call_args.kwargs + assert kwargs["provider"] == "codex_cli" + assert kwargs["model"] == "gpt-5.6-sol" + assert kwargs["input_tokens"] == 10_000 # uncached remainder + assert kwargs["cached_input_tokens"] == 90_000 # cache-READ column + assert kwargs["output_tokens"] == 1_000 + + # 10k uncached @ $5 + 1k out @ $30 + 90k cached @ $0.50 (per Mtok). + expected = 10_000 / 1e6 * 5 + 1_000 / 1e6 * 30 + 90_000 / 1e6 * 0.5 + assert cost_cb.call_args.args[1] == pytest.approx(expected) + assert ss.usage.total_cost_usd == pytest.approx(expected) + # Accumulation saw the disjoint split too (context gauge correctness). + assert ss.usage.input_tokens == 10_000 + assert ss.usage.cache_read_tokens == 90_000 + + # ── gated integration smoke (the make-or-break; opt-in) ───────────────────── @pytest.mark.asyncio @pytest.mark.skipif( diff --git a/tests/test_pricing.py b/tests/test_pricing.py index 8efb087f..69866d43 100644 --- a/tests/test_pricing.py +++ b/tests/test_pricing.py @@ -183,3 +183,79 @@ def test_cost_from_usage_tolerates_malformed() -> None: def test_zero_usage_is_zero_cost() -> None: assert compute_cost_from_usage("claude-opus-4-8", {}) == 0.0 + + +# ── OpenAI / Codex family (#860) ─────────────────────────────────────────── +# Powers the live tmux cost path for codex agents (CodexTmuxSession rides the +# same _log_turn_cost_and_analytics as Claude tmux). Official API rates +# (developers.openai.com, verified 2026-07-10): gpt-5.6-sol / gpt-5.5 at +# $5/$30 (cached input $0.50), gpt-5.3-codex at $1.75/$14 (cached $0.175). +# Cache-write tariffs differ per model page: sol bills 1.25x input ($6.25); +# the gpt-5.5 and gpt-5.3-codex pages list no write tariff (murzik #861 P2). + + +def test_gpt_frontier_rates() -> None: + # 1M input @ $5 + 1M output @ $30 + 1M cached-read @ $0.50 = $35.50 + for model in ("gpt-5.6-sol", "gpt-5.5"): + cost = compute_turn_cost_usd( + model, + input_tokens=1_000_000, + output_tokens=1_000_000, + cache_read_tokens=1_000_000, + cache_creation_5m_tokens=0, + cache_creation_1h_tokens=0, + ) + assert cost == pytest.approx(35.5), model + + +def test_gpt_53_codex_rates() -> None: + # 1M input @ $1.75 + 1M output @ $14 + 1M cached-read @ $0.175 = $15.925 + cost = compute_turn_cost_usd( + "gpt-5.3-codex", + input_tokens=1_000_000, + output_tokens=1_000_000, + cache_read_tokens=1_000_000, + cache_creation_5m_tokens=0, + cache_creation_1h_tokens=0, + ) + assert cost == pytest.approx(15.925) + + +def test_gpt_56_sol_cache_write_billed_at_1_25x_input() -> None: + """The sol model page documents cache writes at 1.25x the uncached input + rate ($6.25/Mtok). OpenAI has no 5m/1h TTL split, so both positions carry + the same tariff (murzik #861 P2 — was wrongly $0).""" + rate = lookup_rate("gpt-5.6-sol") + assert rate["cache_write_5m"] == 6.25 + assert rate["cache_write_1h"] == 6.25 + cost = compute_turn_cost_usd( + "gpt-5.6-sol", + input_tokens=0, + output_tokens=0, + cache_read_tokens=0, + cache_creation_5m_tokens=1_000_000, + cache_creation_1h_tokens=1_000_000, + ) + assert cost == pytest.approx(12.5) + # Usage-dict entry point (aggregate falls back to the 1h position). + usage = {"input_tokens": 0, "output_tokens": 0, + "cache_creation_input_tokens": 1_000_000} + assert compute_cost_from_usage("gpt-5.6-sol", usage) == pytest.approx(6.25) + + +def test_gpt_55_and_codex_cache_write_bills_zero() -> None: + """The gpt-5.5 and gpt-5.3-codex model pages list no cache-write tariff — + cache_creation tokens contribute $0 at both positions.""" + for model in ("gpt-5.5", "gpt-5.3-codex"): + cost = compute_turn_cost_usd( + model, + input_tokens=0, + output_tokens=0, + cache_read_tokens=0, + cache_creation_5m_tokens=1_000_000, + cache_creation_1h_tokens=1_000_000, + ) + assert cost == 0.0, model + usage = {"input_tokens": 0, "output_tokens": 0, + "cache_creation_input_tokens": 1_000_000} + assert compute_cost_from_usage(model, usage) == 0.0, model diff --git a/tests/test_tmux_session.py b/tests/test_tmux_session.py index c014de58..acda94c8 100644 --- a/tests/test_tmux_session.py +++ b/tests/test_tmux_session.py @@ -5419,6 +5419,83 @@ async def test_cost_callback_failure_is_swallowed() -> None: assert ss.usage.total_turns == 1 +# ── #860 — transport-truthful provider attribution + usage normalization ── + + +def _make_usage_session_of( + cls, *, analytics=None, cost_cb=None, model: str = "claude-opus-4-8" +) -> TmuxSession: + """_make_usage_session for a TmuxSession SUBCLASS (transport variants).""" + cfg = StreamingSessionConfig( + agent_name="dymok", + working_dir="/tmp/tmux-session-test", + model=model, + ) + ss = cls( + cfg, + tmux_control=_make_mock_tmux(), + analytics_store=analytics, + cost_callback=cost_cb, + ) + ss._skip_wake_prompt_for_tests = True + ss._state_machine._state = SessionState.CONNECTED + return ss + + +@pytest.mark.asyncio +async def test_analytics_provider_comes_from_class_attr() -> None: + """#860: the provider on analytics rows must read _ANALYTICS_PROVIDER — + not a hardcoded literal — so transport subclasses (CodexTmuxSession) + attribute their turns truthfully. The hardcoded "anthropic" landed every + codex turn as anthropic/gpt-*, which no pricing row can match ($0).""" + + class _OtherTransport(TmuxSession): + _ANALYTICS_PROVIDER = "other_provider" + + analytics = MagicMock() + ss = _make_usage_session_of(_OtherTransport, analytics=analytics) + _seed_inflight(ss) + await ss._handle_turn_complete(_usage_turn_response()) + assert analytics.log_turn_usage.call_args.kwargs["provider"] == "other_provider" + + +def test_base_normalize_turn_usage_is_identity() -> None: + """The Claude transport already emits the daemon's disjoint convention, + so the base hook must pass the dict through untouched (same object).""" + u = {"input_tokens": 10_000, "cache_read_input_tokens": 2_000} + assert TmuxSession._normalize_turn_usage(u) is u + + +@pytest.mark.asyncio +async def test_normalize_hook_runs_before_all_consumers() -> None: + """#860: _normalize_turn_usage must rewrite response.usage ONCE, before + ANY consumer — accumulation (SessionUsage), pricing, and analytics all + see the normalized dict. A partial conversion spread across consumers is + how the cached split got silently dropped in the first place.""" + + class _NormalizingTransport(TmuxSession): + @staticmethod + def _normalize_turn_usage(u: dict) -> dict: + out = dict(u) + out["input_tokens"] = 7_777 + return out + + analytics = MagicMock() + cost_cb = MagicMock() + ss = _make_usage_session_of( + _NormalizingTransport, analytics=analytics, cost_cb=cost_cb + ) + _seed_inflight(ss) + await ss._handle_turn_complete(_usage_turn_response(input_tokens=999)) + # Analytics row: normalized, not the raw 999. + assert analytics.log_turn_usage.call_args.kwargs["input_tokens"] == 7_777 + # SessionUsage accumulation: normalized. + assert ss.usage.input_tokens == 7_777 + # Cost: priced off the normalized count (opus $5/Mtok input). + expected = 7_777 / 1e6 * 5 + 500 / 1e6 * 25 + 2_000 / 1e6 * 0.5 + assert cost_cb.call_args.args[1] == pytest.approx(expected) + + @pytest.mark.asyncio async def test_emits_context_usage_event_with_sdk_shape(monkeypatch) -> None: """``context_usage`` SSE event must carry the SDK-compatible fields