Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added specs/img/860-codex-cost-before-after.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
7 changes: 6 additions & 1 deletion src/pinky_daemon/agent_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -3966,7 +3966,8 @@ def delete_bot_token(self, token_id: str) -> bool:
("anthropic", "claude-opus-4-5", "Claude Opus 4.5", "Previous-gen Opus.", "opus", 200_000, 0, 5.0, 25.0, 0.5, 1, 40),
("anthropic", "claude-sonnet-4-5", "Claude Sonnet 4.5", "Previous-gen Sonnet.", "sonnet", 200_000, 0, 3.0, 15.0, 0.3, 1, 50),
# OpenAI / Codex CLI
("openai", "gpt-5.5", "GPT-5.5", "Newest frontier. Coding + reasoning. Codex sign-in auth only (API pending).", "flagship", 200_000, 0, 1.75, 14.0, 0.175, 0, 55),
("openai", "gpt-5.6-sol", "GPT-5.6 Sol", "Current codex fleet model (2026-07). Frontier coding + reasoning. Codex sign-in auth only (API pending).", "flagship", 200_000, 0, 5.0, 30.0, 0.5, 0, 54),
("openai", "gpt-5.5", "GPT-5.5", "Previous frontier. Coding + reasoning. Codex sign-in auth only (API pending).", "flagship", 200_000, 0, 5.0, 30.0, 0.5, 0, 55),
("openai", "gpt-5.4", "GPT-5.4", "Flagship. Complex reasoning & coding.", "flagship", 200_000, 0, 1.75, 14.0, 0.175, 0, 60),
("openai", "gpt-5.4-mini", "GPT-5.4 Mini", "Fast + capable. Daily driver.", "mid", 200_000, 0, 0.25, 2.0, 0.025, 0, 70),
("openai", "gpt-5.4-nano", "GPT-5.4 Nano", "Cheapest. High-volume tasks.", "low", 200_000, 0, 0.05, 0.4, 0.005, 0, 80),
Expand All @@ -3986,6 +3987,10 @@ def delete_bot_token(self, token_id: str) -> bool:
("anthropic/claude-opus-4-6", (15.0, 75.0, 1.5), (5.0, 25.0, 0.5)),
("anthropic/claude-opus-4-5", (15.0, 75.0, 1.5), (5.0, 25.0, 0.5)),
("anthropic/claude-haiku-4-5", (0.8, 4.0, 0.08), (1.0, 5.0, 0.1)),
# #860: gpt-5.5 was seeded at the gpt-5.2-tier $1.75/$14 while the
# official rate is $5/$30 (cached $0.50) — analytics_store and
# pricing.RATE_TABLE always had it right; this realigns the catalog.
("openai/gpt-5.5", (1.75, 14.0, 0.175), (5.0, 30.0, 0.5)),
]

def _seed_models(self) -> None:
Expand Down
55 changes: 55 additions & 0 deletions src/pinky_daemon/analytics_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,7 @@ def _init_db(self) -> None:
self._migrate_opus_haiku_seed_pricing(conn)
self._migrate_legacy_dotted_model_ids(conn)
self._ensure_pricing_rows(conn)
self._migrate_codex_provider_attribution(conn)
# Schema migration: add user_message_snippet to turn_usage
cols = {
r[1] for r in conn.execute("PRAGMA table_info(analytics_turn_usage)").fetchall()
Expand Down Expand Up @@ -258,6 +259,11 @@ def _init_db(self) -> None:
_LATEST_PRICING_ADDITIONS = [
("openai", "gpt-5.5", "2020-01-01T00:00:00Z", None, 5.00, 30.00, 0.50, "seed"),
("openai", "gpt-5.3-codex", "2020-01-01T00:00:00Z", None, 1.75, 14.00, 0.175, "seed"),
# gpt-5.6-sol (#860): codex fleet model since 2026-07-09. Official
# rates identical to gpt-5.5 (developers.openai.com, verified
# 2026-07-10). Mirrors pricing.RATE_TABLE (_GPT_FRONTIER), parity-pinned
# by test_seed_pricing_matches_rate_table.
("openai", "gpt-5.6-sol", "2020-01-01T00:00:00Z", None, 5.00, 30.00, 0.50, "seed"),
# Sonnet 5 (2026-06): the current Sonnet tier. Standard $3/$15 (cached
# $0.30) — mirrors pinky_daemon.pricing.RATE_TABLE (_SONNET), pinned by
# test_seed_pricing_matches_rate_table. Introductory $2/$10 runs through
Expand Down Expand Up @@ -470,6 +476,55 @@ def _ensure_pricing_rows(self, conn) -> None:
(*r, r[0], r[1]),
)

def _migrate_codex_provider_attribution(self, conn) -> None:
"""Reattribute + reprice codex turns mislogged as ``anthropic`` (#860).

``TmuxSession._log_turn_cost_and_analytics`` hardcoded
``provider="anthropic"``, so every CodexTmuxSession turn landed as
``anthropic/gpt-*`` — a (provider, model) pair the pricing lookup can
never match (``_provider_alias`` never crosses providers), pricing all
codex history at $0. Heal deployed rows to ``codex_cli`` — the value
new codex turns record and the SDK path's default — which aliases onto
the openai rate rows at read time.

The provider flip alone would over-bill history: pre-#860 rows also
predate ``_normalize_turn_usage``, so their ``input_tokens`` is
codex's INCLUSIVE count with the cached span invisible
(``cached_input_tokens=0`` — the old extraction read only the
Anthropic cache keys). Repricing that at the full input rate
overstates ~7x (measured across the fleet's 2026-07 rollouts: 97.5%
of codex input is cached reads). The per-turn split is unrecoverable
from the DB, so rows carrying that signature move the whole count to
the cached column: error bounded at roughly -20% (the true-uncached
share priced at 10x the cached rate), failing toward the visible
undercount — the same direction ``_normalize_turn_usage`` chooses.

Follows the ``_migrate_opus_haiku_seed_pricing`` precedent: idempotent
(both predicates match only misattributed ``anthropic`` pairs, gone
after the first run) and narrow (a ``gpt-*`` model under ``anthropic``
cannot be legitimate — Anthropic serves no gpt model — so no honest
row can match). Touches turn usage AND session facts so per-session
rollups group consistently.
"""
conn.execute(
"""
UPDATE analytics_turn_usage
SET provider = 'codex_cli',
cached_input_tokens = input_tokens,
input_tokens = 0
WHERE provider = 'anthropic' AND model LIKE 'gpt-%'
AND cached_input_tokens = 0
"""
)
for table in ("analytics_turn_usage", "analytics_session_facts"):
conn.execute(
f"""
UPDATE {table}
SET provider = 'codex_cli'
WHERE provider = 'anthropic' AND model LIKE 'gpt-%'
"""
)

def ensure_session_fact(
self,
*,
Expand Down
37 changes: 37 additions & 0 deletions src/pinky_daemon/codex_tmux_session.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,43 @@ async def paste_text(
class CodexTmuxSession(TmuxSession):
"""Interactive codex-CLI REPL in a detached tmux pane (see module docstring)."""

# #860: analytics rows must not claim "anthropic" for codex turns — the
# pricing lookup keys on (provider, model) and never crosses providers, so
# anthropic/gpt-* priced every turn at $0. "codex_cli" matches the SDK
# path's default (CodexSession._analytics_log_turn_usage) and
# analytics_store._provider_alias maps it onto the openai rate rows.
_ANALYTICS_PROVIDER = "codex_cli"

@staticmethod
def _normalize_turn_usage(u: dict) -> dict:
"""Codex usage schema → the daemon's disjoint convention (#860).

Codex reports ``input_tokens`` INCLUSIVE of the cached prefix, with the
cached span under ``cached_input_tokens`` — while every consumer here
(SessionUsage accumulation, ``compute_cost_from_usage``, analytics
``log_turn_usage``, the context gauge's window reconstruction) follows
the Anthropic convention: ``input_tokens`` is the uncached remainder
and the cached span rides ``cache_read_input_tokens``. Mirrors
``CodexSession.uncached_input_tokens`` on the SDK path; pricing the
inclusive number at the full input rate would overstate review-heavy
sessions roughly 10x. The ambiguous source key is consumed, not
forwarded. A dict without ``cached_input_tokens`` (or with malformed
counts) passes through untouched — fail toward the visible undercount,
never a silent double-bill.
"""
if "cached_input_tokens" not in u:
return u
try:
cached = max(0, int(u.get("cached_input_tokens") or 0))
inclusive = max(0, int(u.get("input_tokens") or 0))
except (TypeError, ValueError):
return u
out = dict(u)
out.pop("cached_input_tokens", None)
out["input_tokens"] = max(0, inclusive - cached)
out["cache_read_input_tokens"] = cached
return out

def __init__(
self,
config: StreamingSessionConfig,
Expand Down
39 changes: 39 additions & 0 deletions src/pinky_daemon/pricing.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,39 @@
"cache_write_5m": 1.00,
"cache_write_1h": 1.60,
}
# OpenAI / Codex family (#860): powers the live tmux cost path for codex
# agents (CodexTmuxSession rides the same _log_turn_cost_and_analytics as
# Claude tmux). Official OpenAI API rates (developers.openai.com, verified
# 2026-07-10); codex agents run on sign-in subscription, so as with CC
# subscription agents these are notional usage-value figures (#648).
# Cache-write billing DIFFERS per model page: gpt-5.6-sol documents cache
# writes at 1.25x the uncached input rate ($6.25/Mtok); the gpt-5.5 and
# gpt-5.3-codex pages list no write tariff, so those bill $0. OpenAI has no
# 5m/1h TTL split — the documented tariff goes in both positions. Cached
# reads use the published cached-input rate. Must mirror the analytics
# openai seeds (test_seed_pricing_matches_rate_table, #669 — that guard
# pins input/output/cache_read; the seed table has no write column).
_GPT_56_SOL = {
"input": 5.00,
"output": 30.00,
"cache_read": 0.50,
"cache_write_5m": 6.25, # 1.25x input per the sol model page
"cache_write_1h": 6.25,
}
_GPT_55 = {
"input": 5.00,
"output": 30.00,
"cache_read": 0.50,
"cache_write_5m": 0.0,
"cache_write_1h": 0.0,
}
_GPT_53_CODEX = {
"input": 1.75,
"output": 14.00,
"cache_read": 0.175,
"cache_write_5m": 0.0,
"cache_write_1h": 0.0,
}

# Bare-model-id → rate dict. Add new model ids here on each release.
RATE_TABLE: dict[str, dict[str, float]] = {
Expand All @@ -117,6 +150,12 @@
"claude-sonnet-4": _SONNET,
"claude-haiku-4-5": _HAIKU_45,
"claude-haiku-3-5": _HAIKU_35,
# Codex fleet models (#860). gpt-5.6-sol is the current fleet model
# (2026-07-09→); gpt-5.5 the previous one; gpt-5.3-codex kept because the
# analytics seed carries it (parity-pinned).
"gpt-5.6-sol": _GPT_56_SOL,
"gpt-5.5": _GPT_55,
"gpt-5.3-codex": _GPT_53_CODEX,
}


Expand Down
30 changes: 29 additions & 1 deletion src/pinky_daemon/tmux_session.py
Original file line number Diff line number Diff line change
Expand Up @@ -3804,6 +3804,27 @@ def _soft_nudge_threshold_pct(self) -> float:
pass
return DEFAULT_CONTEXT_NUDGE_THRESHOLD_PCT

# Provider recorded on analytics_turn_usage rows (#860). Class-level so
# transport subclasses that ride this same cost/analytics path attribute
# their turns truthfully — CodexTmuxSession overrides with "codex_cli"
# (analytics_store._provider_alias maps it to the openai rate rows).
_ANALYTICS_PROVIDER: str = "anthropic"

@staticmethod
def _normalize_turn_usage(u: dict) -> dict:
"""Transport-specific usage normalization hook (#860).

The base (Claude) transport already emits the daemon's disjoint
convention, so this is the identity. CodexTmuxSession overrides it to
convert the codex schema (``input_tokens`` INCLUSIVE of the cached
prefix, cached span under ``cached_input_tokens``) before any
consumer — usage accumulation, pricing, analytics, context gauge —
reads the dict. ONE conversion point by design: a partial conversion
spread across consumers is how the cached split got silently dropped
in the first place.
"""
return u

def _record_turn_usage(self, response: TurnResponse) -> None:
"""Fold a turn's usage block into ``self.usage`` (SessionUsage).

Expand Down Expand Up @@ -3929,7 +3950,10 @@ def _log_turn_cost_and_analytics(self, response: TurnResponse) -> None:
session_id=self.id,
agent_name=self.agent_name,
turn_seq=turn_seq,
provider="anthropic",
# #860: was hardcoded "anthropic", which mispriced every
# CodexTmuxSession turn to $0 (anthropic/gpt-* never
# matches the openai rate rows).
provider=self._ANALYTICS_PROVIDER,
model=model,
input_tokens=input_tokens,
output_tokens=output_tokens,
Expand Down Expand Up @@ -4380,6 +4404,10 @@ async def _handle_turn_complete(self, response: TurnResponse) -> None:
# tailer already pulled the usage dict out of each assistant
# entry's ``usage`` block; we just need to fold it into the
# session-level dataclass and emit it.
# #860: normalize transport-specific usage schemas ONCE, before any
# consumer (accumulation, pricing, analytics, context gauge) reads it.
if isinstance(response.usage, dict):
response.usage = self._normalize_turn_usage(response.usage)
self._record_turn_usage(response)
# #648 — forward per-turn usage to analytics + cost tracking so
# tmux agents reach live Analytics / lifetime-cost parity with the
Expand Down
62 changes: 62 additions & 0 deletions tests/test_agent_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -995,3 +995,65 @@ def test_operator_customized_prices_survive_reseed(self, registry):
if m["id"] == "anthropic/claude-opus-4-8"
)
assert opus["input_price"] == 12.34

def test_gpt_56_sol_seeded_at_frontier_rates(self, registry):
"""#860: gpt-5.6-sol (the codex fleet model since 2026-07) must be in
the catalog at the official $5/$30 (cached $0.50)."""
models = {
m["model_id"]: m
for m in registry.list_models(provider="openai", active_only=False)
}
assert "gpt-5.6-sol" in models
sol = models["gpt-5.6-sol"]
assert (sol["input_price"], sol["output_price"],
sol["cached_input_price"]) == (5.0, 30.0, 0.5)
assert sol["tier"] == "flagship"

def test_openai_seed_prices_match_pricing_rate_table(self, registry):
"""#860 extends the #741 invariant to the OpenAI family: catalog
display prices must agree with pricing.py (the actual cost engine)
for every openai model both tables know."""
from pinky_daemon.pricing import RATE_TABLE

checked = 0
for m in registry.list_models(provider="openai", active_only=False):
rate = RATE_TABLE.get(m["model_id"])
if rate is None:
continue
assert m["input_price"] == rate["input"], m["model_id"]
assert m["output_price"] == rate["output"], m["model_id"]
assert m["cached_input_price"] == rate["cache_read"], m["model_id"]
checked += 1
# Guard the guard: gpt-5.6-sol + gpt-5.5 must both be intersecting.
assert checked >= 2

def test_stale_gpt55_price_corrected_on_existing_rows(self, registry):
"""#860: deployed DBs seeded gpt-5.5 at the gpt-5.2-tier $1.75/$14;
the next seed pass realigns existing rows to the official $5/$30
(INSERT OR IGNORE alone never reaches them)."""
registry._db.execute(
"UPDATE models SET input_price=1.75, output_price=14.0,"
" cached_input_price=0.175 WHERE id='openai/gpt-5.5'"
)
registry._db.commit()
registry._seed_models()
gpt = next(
m for m in registry.list_models(provider="openai", active_only=False)
if m["id"] == "openai/gpt-5.5"
)
assert (gpt["input_price"], gpt["output_price"],
gpt["cached_input_price"]) == (5.0, 30.0, 0.5)

def test_operator_customized_gpt55_price_survives_reseed(self, registry):
"""Same exact-stale-triple gate as the Anthropic corrections — an
operator-priced gpt-5.5 row must not be clobbered."""
registry._db.execute(
"UPDATE models SET input_price=9.99 WHERE id='openai/gpt-5.5'"
)
registry._db.commit()
registry._seed_models()
gpt = next(
m for m in registry.list_models(provider="openai", active_only=False)
if m["id"] == "openai/gpt-5.5"
)
assert gpt["input_price"] == 9.99
Loading
Loading