From ff35050e3b14b02c065f07967093df7205daccbc Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Wed, 5 Aug 2026 18:00:55 -0700 Subject: [PATCH 01/15] Re-express UsageRow in OTel GenAI semantic conventions Replace the flat UsageRow with an OTel-GenAI-shaped row: gen_ai.* dotted attribute keys plus a thirdeye envelope. Key changes: - Inclusive input_tokens; optional cache/reasoning fields are int | None, serialized only when present (absent-vs-zero distinction). - total_tokens is now a derived property, never stored or serialized. - attributes() emits only gen_ai.* keys (incl. gen_ai.conversation.id) as the future OTLP exporter surface; to_dict() merges the envelope. - ATTRIBUTE_KEYS maps fields to the exact spec names, no abbreviations. Tests cover round-trip, None-omission, zero-preservation, conformance to the nine literal spec names, and envelope/attribute separation. Co-Authored-By: Claude Opus 4.8 --- src/thirdeye/usage/types.py | 93 +++++++++++++++++++--- tests/test_usage_types.py | 155 +++++++++++++++++++++++++++++------- 2 files changed, 209 insertions(+), 39 deletions(-) diff --git a/src/thirdeye/usage/types.py b/src/thirdeye/usage/types.py index 8a0136a..b514371 100644 --- a/src/thirdeye/usage/types.py +++ b/src/thirdeye/usage/types.py @@ -1,34 +1,109 @@ from __future__ import annotations -from dataclasses import asdict, dataclass +from dataclasses import dataclass from typing import Any +# Mapping from dataclass field name to its dotted OpenTelemetry GenAI spec key. +# All gen_ai.* attributes are Development status (moved to a dedicated repo in +# semconv v1.42.0). These are the exact spec names — no abbreviations, no +# invented attributes. +ATTRIBUTE_KEYS: dict[str, str] = { + "provider_name": "gen_ai.provider.name", + "operation_name": "gen_ai.operation.name", + "response_model": "gen_ai.response.model", + "input_tokens": "gen_ai.usage.input_tokens", + "output_tokens": "gen_ai.usage.output_tokens", + "cache_read_input_tokens": "gen_ai.usage.cache_read.input_tokens", + "cache_creation_input_tokens": "gen_ai.usage.cache_creation.input_tokens", + "reasoning_output_tokens": "gen_ai.usage.reasoning.output_tokens", +} + +# The thirdeye envelope fields — NOT gen_ai attributes. Serialized as bare names. +_ENVELOPE_KEYS: tuple[str, ...] = ("session_id", "seq", "call_id", "ts", "platform") + +# gen_ai.conversation.id is always emitted (mirrors session_id) but is not a +# stored dataclass field, so it lives outside ATTRIBUTE_KEYS. +_CONVERSATION_ID_KEY = "gen_ai.conversation.id" + +# Optional gen_ai.* fields, absent-vs-zero: an unreported attribute is an absent +# key (None), never 0. +_OPTIONAL_FIELDS: tuple[str, ...] = ( + "cache_read_input_tokens", + "cache_creation_input_tokens", + "reasoning_output_tokens", +) + @dataclass(frozen=True) class UsageRow: - """One row of model + token usage data, joinable back to events.alog by seq.""" + """One model call's usage, shaped as OpenTelemetry GenAI semantic conventions. + + Serializes to dotted ``gen_ai.*`` spec keys plus a small thirdeye envelope. + There is no ``total_tokens`` attribute in the spec — it is a derived property, + never stored or serialized. + """ + # --- thirdeye envelope (NOT gen_ai attributes) --- session_id: str seq: int + call_id: str ts: str platform: str - model: str + # --- gen_ai.* attributes --- + provider_name: str + response_model: str input_tokens: int output_tokens: int - total_tokens: int + operation_name: str = "chat" + cache_read_input_tokens: int | None = None + cache_creation_input_tokens: int | None = None + reasoning_output_tokens: int | None = None + + @property + def total_tokens(self) -> int: + """Derived, never stored or serialized.""" + return self.input_tokens + self.output_tokens + + def attributes(self) -> dict[str, Any]: + """Only the gen_ai.* attributes, omitting any whose value is None. + + Always includes gen_ai.conversation.id (mirrors session_id). This method + is the whole future OTLP exporter; keep it pure. + """ + out: dict[str, Any] = {_CONVERSATION_ID_KEY: self.session_id} + for field_name, key in ATTRIBUTE_KEYS.items(): + value = getattr(self, field_name) + if value is None: + continue + out[key] = value + return out def to_dict(self) -> dict[str, Any]: - return asdict(self) + """Envelope keys (bare names) merged with attributes().""" + out: dict[str, Any] = {name: getattr(self, name) for name in _ENVELOPE_KEYS} + out.update(self.attributes()) + return out @classmethod def from_dict(cls, d: dict[str, Any]) -> UsageRow: + """Inverse of to_dict. Missing optional attribute keys become None. + + Raises KeyError / ValueError on a malformed row. + """ + optional: dict[str, int | None] = {} + for field_name in _OPTIONAL_FIELDS: + key = ATTRIBUTE_KEYS[field_name] + optional[field_name] = int(d[key]) if key in d else None return cls( session_id=str(d["session_id"]), seq=int(d["seq"]), + call_id=str(d["call_id"]), ts=str(d["ts"]), platform=str(d["platform"]), - model=str(d["model"]), - input_tokens=int(d["input_tokens"]), - output_tokens=int(d["output_tokens"]), - total_tokens=int(d["total_tokens"]), + provider_name=str(d[ATTRIBUTE_KEYS["provider_name"]]), + response_model=str(d[ATTRIBUTE_KEYS["response_model"]]), + input_tokens=int(d[ATTRIBUTE_KEYS["input_tokens"]]), + output_tokens=int(d[ATTRIBUTE_KEYS["output_tokens"]]), + operation_name=str(d[ATTRIBUTE_KEYS["operation_name"]]), + **optional, ) diff --git a/tests/test_usage_types.py b/tests/test_usage_types.py index 7bdc897..d9fd9aa 100644 --- a/tests/test_usage_types.py +++ b/tests/test_usage_types.py @@ -4,64 +4,159 @@ import pytest -from thirdeye.usage.types import UsageRow +from thirdeye.usage.types import ATTRIBUTE_KEYS, UsageRow + +# The nine gen_ai.* spec names, written as literals. Do NOT derive this from +# ATTRIBUTE_KEYS — a hardcoded set is what makes the conformance test meaningful. +SPEC_GEN_AI_KEYS = { + "gen_ai.provider.name", + "gen_ai.operation.name", + "gen_ai.conversation.id", + "gen_ai.response.model", + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + "gen_ai.usage.cache_read.input_tokens", + "gen_ai.usage.cache_creation.input_tokens", + "gen_ai.usage.reasoning.output_tokens", +} + +ENVELOPE_KEYS = {"session_id", "seq", "call_id", "ts", "platform"} def make_row(**overrides) -> UsageRow: defaults = dict( session_id="abc123", seq=0, + call_id="msg_001", ts="2026-05-15T00:00:00.000Z", platform="claude", - model="claude-opus-4-7", + provider_name="anthropic", + response_model="claude-opus-4-8", input_tokens=100, output_tokens=10, - total_tokens=110, ) defaults.update(overrides) return UsageRow(**defaults) -def test_round_trip_via_dict() -> None: +def full_row() -> UsageRow: + return make_row( + cache_read_input_tokens=50, + cache_creation_input_tokens=25, + reasoning_output_tokens=5, + ) + + +def test_round_trip_all_optionals_set() -> None: + row = full_row() + assert UsageRow.from_dict(row.to_dict()) == row + + +def test_round_trip_all_optionals_none() -> None: row = make_row() + assert row.cache_read_input_tokens is None assert UsageRow.from_dict(row.to_dict()) == row def test_round_trip_via_json() -> None: - row = make_row() - encoded = json.dumps(row.to_dict()) - decoded = UsageRow.from_dict(json.loads(encoded)) + row = full_row() + decoded = UsageRow.from_dict(json.loads(json.dumps(row.to_dict()))) assert decoded == row -def test_from_dict_coerces_string_numerics() -> None: - """Integer-like strings should coerce, mirroring JSON-from-disk quirks.""" - row = UsageRow.from_dict( - { - "session_id": "abc", - "seq": "5", - "ts": "2026-05-15T00:00:00Z", - "platform": "claude", - "model": "m", - "input_tokens": "100", - "output_tokens": "10", - "total_tokens": "110", - } - ) - assert row.seq == 5 and row.input_tokens == 100 +def test_to_dict_omits_none_optionals() -> None: + d = make_row().to_dict() + assert "gen_ai.usage.cache_read.input_tokens" not in d + assert "gen_ai.usage.cache_creation.input_tokens" not in d + assert "gen_ai.usage.reasoning.output_tokens" not in d + + +def test_zero_optional_serializes_as_zero() -> None: + """absent-vs-zero: 0 means 'reported as none', which must be kept.""" + d = make_row(cache_read_input_tokens=0).to_dict() + assert d["gen_ai.usage.cache_read.input_tokens"] == 0 + assert UsageRow.from_dict(d).cache_read_input_tokens == 0 -def test_from_dict_missing_field_raises() -> None: +def test_attributes_only_gen_ai_keys() -> None: + attrs = full_row().attributes() + assert all(k.startswith("gen_ai.") for k in attrs) + + +def test_attributes_includes_conversation_id() -> None: + row = make_row(session_id="sess-xyz") + assert row.attributes()["gen_ai.conversation.id"] == "sess-xyz" + + +def test_attributes_has_no_envelope_keys() -> None: + attrs = full_row().attributes() + assert not (ENVELOPE_KEYS & set(attrs)) + + +def test_total_tokens_is_derived() -> None: + row = make_row(input_tokens=100, output_tokens=10) + assert row.total_tokens == 110 + d = row.to_dict() + assert "total_tokens" not in d + assert "gen_ai.usage.total_tokens" not in d + + +def test_expected_claude_serialization() -> None: + row = UsageRow( + session_id="acb30f50", + seq=36, + call_id="msg_011Cd", + ts="2026-08-05T16:53:52.790Z", + platform="claude", + provider_name="anthropic", + response_model="claude-opus-4-8", + input_tokens=324429, + output_tokens=1230, + cache_read_input_tokens=304110, + cache_creation_input_tokens=20317, + ) + assert row.to_dict() == { + "session_id": "acb30f50", + "seq": 36, + "call_id": "msg_011Cd", + "ts": "2026-08-05T16:53:52.790Z", + "platform": "claude", + "gen_ai.provider.name": "anthropic", + "gen_ai.operation.name": "chat", + "gen_ai.conversation.id": "acb30f50", + "gen_ai.response.model": "claude-opus-4-8", + "gen_ai.usage.input_tokens": 324429, + "gen_ai.usage.output_tokens": 1230, + "gen_ai.usage.cache_read.input_tokens": 304110, + "gen_ai.usage.cache_creation.input_tokens": 20317, + } + + +def test_conformance_only_spec_gen_ai_keys() -> None: + """Every gen_ai.* key emitted must be one of the nine literal spec names. + + A typo or invented attribute must fail here. + """ + d = full_row().to_dict() + gen_ai_keys = {k for k in d if k.startswith("gen_ai.")} + assert gen_ai_keys <= SPEC_GEN_AI_KEYS + # The full row exercises all nine. + assert gen_ai_keys == SPEC_GEN_AI_KEYS + + +def test_attribute_keys_map_matches_spec() -> None: + """ATTRIBUTE_KEYS values are a subset of the spec names (all but conversation.id).""" + assert set(ATTRIBUTE_KEYS.values()) == SPEC_GEN_AI_KEYS - {"gen_ai.conversation.id"} + + +def test_from_dict_missing_session_id_raises() -> None: + d = full_row().to_dict() + del d["session_id"] with pytest.raises(KeyError): - UsageRow.from_dict({"session_id": "abc"}) + UsageRow.from_dict(d) def test_is_frozen() -> None: row = make_row() - with pytest.raises(Exception): # FrozenInstanceError, but version-dependent + with pytest.raises(Exception): # FrozenInstanceError, version-dependent row.seq = 99 # type: ignore[misc] - - -def test_equality_value_based() -> None: - assert make_row() == make_row() - assert make_row(seq=1) != make_row(seq=2) From 3c0cb332fb38fdbfce6f773d7d4663f3bbba26ec Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Wed, 5 Aug 2026 18:03:38 -0700 Subject: [PATCH 02/15] Drop Gemini and Cursor tracing platforms Their payload formats were never verified against real data and are replaced in a later plan by Antigravity and a rebuilt Cursor. - Delete src/thirdeye/platforms/{gemini,cursor}/ and their test files - Remove the orphaned gemini_model_response.json fixture - Reduce PLATFORMS registry in commands/add.py to {claude, codex} - Drop the thirdeye-gemini-* and thirdeye-cursor-hook console scripts - Fix ingest --platform help text (cursor -> codex) - Add find_orphaned_hooks() + `thirdeye add --list` so users are warned about now-orphaned hook entries left in ~/.gemini and ~/.cursor config - Rewrite tests/test_add_command.py for the two-platform registry and cover find_orphaned_hooks with tmp_path fixtures Co-Authored-By: Claude Opus 4.8 --- pyproject.toml | 9 - src/thirdeye/commands/add.py | 78 +- src/thirdeye/commands/eval.py | 5 +- src/thirdeye/commands/ingest.py | 2 +- src/thirdeye/commands/ui.py | 2 +- src/thirdeye/commands/usage.py | 10 +- src/thirdeye/eval/definition.py | 2 +- src/thirdeye/eval/prompt.py | 2 +- src/thirdeye/platforms/cursor/__init__.py | 0 src/thirdeye/platforms/cursor/constants.py | 65 -- src/thirdeye/platforms/cursor/hook.py | 239 ----- src/thirdeye/platforms/cursor/install.py | 74 -- src/thirdeye/platforms/cursor/usage.py | 81 -- src/thirdeye/platforms/gemini/__init__.py | 0 src/thirdeye/platforms/gemini/constants.py | 30 - src/thirdeye/platforms/gemini/hooks.py | 209 ----- src/thirdeye/platforms/gemini/install.py | 101 --- src/thirdeye/platforms/gemini/usage.py | 56 -- src/thirdeye/timeparse.py | 2 +- .../fixtures/usage/gemini_model_response.json | 19 - tests/test_add_command.py | 368 +++----- tests/test_cursor_constants.py | 73 -- tests/test_cursor_hook.py | 578 ------------ tests/test_cursor_install.py | 217 ----- tests/test_cursor_usage.py | 245 ------ tests/test_e2e_gemini.py | 298 ------- tests/test_gemini_constants.py | 64 -- tests/test_gemini_hooks.py | 833 ------------------ tests/test_gemini_install.py | 511 ----------- tests/test_usage_claude.py | 2 +- tests/test_usage_gemini.py | 102 --- tests/web/test_evals_session_def_panel.py | 6 +- tests/web/test_routes_evals_read.py | 12 +- tests/web/test_skeleton_invariants.py | 6 +- 34 files changed, 214 insertions(+), 4087 deletions(-) delete mode 100644 src/thirdeye/platforms/cursor/__init__.py delete mode 100644 src/thirdeye/platforms/cursor/constants.py delete mode 100644 src/thirdeye/platforms/cursor/hook.py delete mode 100644 src/thirdeye/platforms/cursor/install.py delete mode 100644 src/thirdeye/platforms/cursor/usage.py delete mode 100644 src/thirdeye/platforms/gemini/__init__.py delete mode 100644 src/thirdeye/platforms/gemini/constants.py delete mode 100644 src/thirdeye/platforms/gemini/hooks.py delete mode 100644 src/thirdeye/platforms/gemini/install.py delete mode 100644 src/thirdeye/platforms/gemini/usage.py delete mode 100644 tests/fixtures/usage/gemini_model_response.json delete mode 100644 tests/test_cursor_constants.py delete mode 100644 tests/test_cursor_hook.py delete mode 100644 tests/test_cursor_install.py delete mode 100644 tests/test_cursor_usage.py delete mode 100644 tests/test_e2e_gemini.py delete mode 100644 tests/test_gemini_constants.py delete mode 100644 tests/test_gemini_hooks.py delete mode 100644 tests/test_gemini_install.py delete mode 100644 tests/test_usage_gemini.py diff --git a/pyproject.toml b/pyproject.toml index 0d5bb39..2dedfb0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,15 +44,6 @@ thirdeye-claude-notification = "thirdeye.platforms.claude.hooks:notification" thirdeye-claude-permission-request = "thirdeye.platforms.claude.hooks:permission_request" thirdeye-claude-session-end = "thirdeye.platforms.claude.hooks:session_end" thirdeye-codex-notify = "thirdeye.platforms.codex.hooks:notify" -thirdeye-gemini-session-start = "thirdeye.platforms.gemini.hooks:session_start" -thirdeye-gemini-session-end = "thirdeye.platforms.gemini.hooks:session_end" -thirdeye-gemini-before-agent = "thirdeye.platforms.gemini.hooks:before_agent" -thirdeye-gemini-after-agent = "thirdeye.platforms.gemini.hooks:after_agent" -thirdeye-gemini-before-model = "thirdeye.platforms.gemini.hooks:before_model" -thirdeye-gemini-after-model = "thirdeye.platforms.gemini.hooks:after_model" -thirdeye-gemini-before-tool = "thirdeye.platforms.gemini.hooks:before_tool" -thirdeye-gemini-after-tool = "thirdeye.platforms.gemini.hooks:after_tool" -thirdeye-cursor-hook = "thirdeye.platforms.cursor.hook:main" [build-system] requires = ["setuptools>=64", "setuptools-scm>=8", "wheel"] diff --git a/src/thirdeye/commands/add.py b/src/thirdeye/commands/add.py index e02f54c..fc8badb 100644 --- a/src/thirdeye/commands/add.py +++ b/src/thirdeye/commands/add.py @@ -1,38 +1,100 @@ from __future__ import annotations +import json +from collections.abc import Iterable +from pathlib import Path + import click from thirdeye.platforms.base import Platform from thirdeye.platforms.claude.install import ClaudePlatform from thirdeye.platforms.codex.install import CodexPlatform -from thirdeye.platforms.cursor.install import CursorPlatform -from thirdeye.platforms.gemini.install import GeminiPlatform PLATFORMS: dict[str, type[Platform]] = { "claude": ClaudePlatform, - "gemini": GeminiPlatform, "codex": CodexPlatform, - "cursor": CursorPlatform, } +# Config files that may still reference console scripts for platforms this +# version no longer installs (Gemini and Cursor). Deleting those platforms +# leaves their hook entries orphaned in other tools' config, firing a missing +# binary on every event, so `thirdeye add --list` warns about them. +ORPHAN_CONFIG_PATHS: tuple[Path, ...] = ( + Path.home() / ".gemini" / "settings.json", + Path.home() / ".cursor" / "hooks.json", +) + def _platform_options(fn): - fn = click.option("--cursor", "platform_flag", flag_value="cursor", help="Cursor.")(fn) fn = click.option("--codex", "platform_flag", flag_value="codex", help="Codex CLI.")(fn) - fn = click.option("--gemini", "platform_flag", flag_value="gemini", help="Gemini CLI.")(fn) fn = click.option("--claude", "platform_flag", flag_value="claude", help="Claude Code.")(fn) return fn def _resolve_platform(platform_flag: str | None) -> Platform: if not platform_flag: - raise click.UsageError("Pick a platform: --claude, --gemini, --codex, --cursor") + raise click.UsageError("Pick a platform: --claude, --codex") return PLATFORMS[platform_flag]() +def _is_stale_command(command: str) -> bool: + """True if a hook command targets a removed-platform console script.""" + name = Path(command).name + return name.startswith("thirdeye-gemini-") or name == "thirdeye-cursor-hook" + + +def find_orphaned_hooks( + config_paths: Iterable[Path] = ORPHAN_CONFIG_PATHS, +) -> list[tuple[Path, str]]: + """Return (config_file, stale_command) for each removed-platform hook found. + + A command is stale when its basename starts with "thirdeye-gemini-" or + equals "thirdeye-cursor-hook". Missing or malformed files yield nothing. + """ + found: list[tuple[Path, str]] = [] + for path in config_paths: + try: + data = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + continue + for command in _iter_commands(data): + if _is_stale_command(command): + found.append((path, command)) + return found + + +def _iter_commands(node: object) -> Iterable[str]: + """Yield every string under a "command" key anywhere in a nested structure.""" + if isinstance(node, dict): + for key, value in node.items(): + if key == "command" and isinstance(value, str): + yield value + else: + yield from _iter_commands(value) + elif isinstance(node, list): + for item in node: + yield from _iter_commands(item) + + @click.command(help="Install tracing hooks for an agentic platform.") +@click.option( + "--list", + "list_platforms", + is_flag=True, + help="List supported platforms and warn about orphaned hooks from removed platforms.", +) @_platform_options -def add(platform_flag: str | None) -> None: +def add(platform_flag: str | None, list_platforms: bool) -> None: + if list_platforms: + click.echo("Supported platforms:") + for name in PLATFORMS: + click.echo(f" {name}") + for path, command in find_orphaned_hooks(ORPHAN_CONFIG_PATHS): + click.echo( + f"Warning: {path} still references removed hook {command!r}. " + "Remove it from that tool's config to stop the missing-binary error.", + ) + return platform = _resolve_platform(platform_flag) platform.install() click.echo(f"Installed tracing for {platform.display_name}") diff --git a/src/thirdeye/commands/eval.py b/src/thirdeye/commands/eval.py index 8ba13dc..b1f27bd 100644 --- a/src/thirdeye/commands/eval.py +++ b/src/thirdeye/commands/eval.py @@ -49,8 +49,7 @@ def run_cmd(session_prefix, using, agent, background, as_json, save): if agent not in list_agent_names(config.root): raise click.ClickException( - f"unknown agent {agent!r} — choose one of " - f"{', '.join(list_agent_names(config.root))}" + f"unknown agent {agent!r} — choose one of {', '.join(list_agent_names(config.root))}" ) if background: @@ -267,7 +266,7 @@ def status_cmd(session_prefix, as_json): click.echo("No background eval jobs.") return click.echo( - f"{'JOB':<28} {'SESSION':<14} {'USING':<18} {'AGENT':<8} " f"{'STATUS':<10} {'STARTED':<26}" + f"{'JOB':<28} {'SESSION':<14} {'USING':<18} {'AGENT':<8} {'STATUS':<10} {'STARTED':<26}" ) for j in jobs: click.echo( diff --git a/src/thirdeye/commands/ingest.py b/src/thirdeye/commands/ingest.py index 56a6d65..dc16d8b 100644 --- a/src/thirdeye/commands/ingest.py +++ b/src/thirdeye/commands/ingest.py @@ -11,7 +11,7 @@ @click.command(help="Read newline-delimited JSON events from stdin and append them to a session.") -@click.option("--platform", required=True, help="Platform name (e.g. claude, cursor).") +@click.option("--platform", required=True, help="Platform name (e.g. claude, codex).") @click.option("--session-id", default=None, help="Session ID. Generated if omitted.") @click.option("--cwd", default=None, help="Working directory for the session.") def ingest(platform: str, session_id: str | None, cwd: str | None) -> None: diff --git a/src/thirdeye/commands/ui.py b/src/thirdeye/commands/ui.py index d5222e0..cb71033 100644 --- a/src/thirdeye/commands/ui.py +++ b/src/thirdeye/commands/ui.py @@ -30,7 +30,7 @@ def ui(host: str, port: int, no_browser: bool) -> None: msg = str(e) if "starlette" in msg or "uvicorn" in msg or "jinja2" in msg: raise click.ClickException( - "The UI requires the 'ui' extra. Install with:\n" " pip install 'thrdi[ui]'" + "The UI requires the 'ui' extra. Install with:\n pip install 'thrdi[ui]'" ) from e raise diff --git a/src/thirdeye/commands/usage.py b/src/thirdeye/commands/usage.py index 5ebf15b..394bf08 100644 --- a/src/thirdeye/commands/usage.py +++ b/src/thirdeye/commands/usage.py @@ -191,9 +191,9 @@ def errors_cmd(n, as_json, platform_filter, phase, since, until): return for e in entries: click.echo( - f"{e.get('ts','')} {e.get('level','?'):<5} " - f"{e.get('platform','?'):<7} {e.get('phase','?'):<20} " - f"{e.get('session_id','')[:12]:<12} {e.get('message','')}" + f"{e.get('ts', '')} {e.get('level', '?'):<5} " + f"{e.get('platform', '?'):<7} {e.get('phase', '?'):<20} " + f"{e.get('session_id', '')[:12]:<12} {e.get('message', '')}" ) @@ -302,7 +302,7 @@ def _render_session( click.echo(f"{'SEQ':<5} {'TS':<26} {'MODEL':<25} {'INPUT':>10} {'OUTPUT':>8} {'TOTAL':>10}") tot_in = tot_out = tot = 0 for r in rows: - click.echo(f"{r[0]:<5} {r[1]:<26} {r[3][:25]:<25} " f"{r[4]:>10,} {r[5]:>8,} {r[6]:>10,}") + click.echo(f"{r[0]:<5} {r[1]:<26} {r[3][:25]:<25} {r[4]:>10,} {r[5]:>8,} {r[6]:>10,}") tot_in += r[4] tot_out += r[5] tot += r[6] @@ -378,7 +378,7 @@ def _render_rollup( ) tot_in = tot_out = tot = 0 for r in rows: - click.echo(f"{r[0][:14]:<14} {r[1]:<9} {r[2]:>5} " f"{r[3]:>12,} {r[4]:>10,} {r[5]:>12,}") + click.echo(f"{r[0][:14]:<14} {r[1]:<9} {r[2]:>5} {r[3]:>12,} {r[4]:>10,} {r[5]:>12,}") tot_in += r[3] tot_out += r[4] tot += r[5] diff --git a/src/thirdeye/eval/definition.py b/src/thirdeye/eval/definition.py index e614814..fb9b510 100644 --- a/src/thirdeye/eval/definition.py +++ b/src/thirdeye/eval/definition.py @@ -58,7 +58,7 @@ def load_definition(thirdeye_home: Path, name: str) -> EvalDefinition: shipped = _shipped_path(name) if shipped is None: raise FileNotFoundError( - f"no eval definition named '{name}' " f"(checked {user_path} and shipped defaults)" + f"no eval definition named '{name}' (checked {user_path} and shipped defaults)" ) user_path.parent.mkdir(parents=True, exist_ok=True) user_path.write_text(shipped.read_text(encoding="utf-8"), encoding="utf-8") diff --git a/src/thirdeye/eval/prompt.py b/src/thirdeye/eval/prompt.py index 5cfc8ee..51f2551 100644 --- a/src/thirdeye/eval/prompt.py +++ b/src/thirdeye/eval/prompt.py @@ -74,7 +74,7 @@ def build_prompt( blocks.append("=== Tool inventory ===") blocks.append("You have read-only access to:") blocks.append( - "- `thirdeye` CLI: list, events, show, tail, event, search, " "tag, tags, stats, usage" + "- `thirdeye` CLI: list, events, show, tail, event, search, tag, tags, stats, usage" ) blocks.append(f"- `sqlite3 {db}`") blocks.append("- `jq`, `Read`") diff --git a/src/thirdeye/platforms/cursor/__init__.py b/src/thirdeye/platforms/cursor/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/thirdeye/platforms/cursor/constants.py b/src/thirdeye/platforms/cursor/constants.py deleted file mode 100644 index aeb3e5b..0000000 --- a/src/thirdeye/platforms/cursor/constants.py +++ /dev/null @@ -1,65 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -PLATFORM_NAME = "cursor" -DISPLAY_NAME = "Cursor" - -HOOKS_FILE = Path.home() / ".cursor" / "hooks.json" - -HOOK_BIN_NAME = "thirdeye-cursor-hook" - -HOOK_TIMEOUT_S = 30 - -TRACED_EVENTS: tuple[str, ...] = ( - "sessionStart", - "sessionEnd", - "beforeSubmitPrompt", - "afterAgentResponse", - "beforeShellExecution", - "afterShellExecution", - "beforeMCPExecution", - "afterMCPExecution", - "beforeReadFile", - "afterFileEdit", - "beforeTabFileRead", - "afterTabFileEdit", - "postToolUse", - "stop", -) - -DEDICATED_TOOL_NAMES: frozenset[str] = frozenset( - { - "shell", - "terminal", - "bash", - "run_command", - "run_shell", - "read_file", - "read", - "view_file", - "view", - "edit_file", - "edit", - "write_file", - "write", - "create_file", - "delete_file", - "tab_file_read", - "tab_file_edit", - "mcp", - "mcp_execution", - } -) - -STRIP_KEYS: frozenset[str] = frozenset( - { - "conversation_id", - "conversationId", - "workspace_roots", - "transcript_path", - "user_email", - "hook_event_name", - "hookEventName", - } -) diff --git a/src/thirdeye/platforms/cursor/hook.py b/src/thirdeye/platforms/cursor/hook.py deleted file mode 100644 index 6b09300..0000000 --- a/src/thirdeye/platforms/cursor/hook.py +++ /dev/null @@ -1,239 +0,0 @@ -from __future__ import annotations - -import json -import os -import sys -from collections.abc import Callable - -from thirdeye.config import Config -from thirdeye.meta import read_meta, write_meta -from thirdeye.paths import meta_path, session_dir -from thirdeye.platforms.cursor.constants import DEDICATED_TOOL_NAMES, STRIP_KEYS -from thirdeye.platforms.cursor.usage import capture_usage_cursor -from thirdeye.store import Store -from thirdeye.tags import TagStore, extract_hashtags - -_PLATFORM = "cursor" - - -def _read_stdin() -> dict: - try: - raw = sys.stdin.read() - except OSError: - return {} - if not raw: - return {} - try: - return json.loads(raw) - except json.JSONDecodeError: - return {} - - -def _get_str(p: dict, *keys: str, default: str = "") -> str: - for k in keys: - v = p.get(k) - if v is not None and v != "": - return str(v) - return default - - -def _event_name(p: dict) -> str: - return _get_str(p, "hook_event_name", "hookEventName") - - -def _session_id(p: dict) -> str: - return _get_str(p, "conversation_id", "conversationId") - - -def _cwd(p: dict) -> str: - cwd = p.get("cwd") - if cwd: - return str(cwd) - roots = p.get("workspace_roots") - if isinstance(roots, list) and roots: - return str(roots[0]) - return os.getcwd() - - -def _strip(p: dict) -> dict: - return {k: v for k, v in p.items() if k not in STRIP_KEYS} - - -def _emit(payload: dict, t: str, extra: dict | None = None) -> int | None: - sid = _session_id(payload) - if not sid: - return None - data = _strip(payload) - if extra: - data = {**data, **extra} - return Store(Config.load()).append_event( - session_id=sid, - platform=_PLATFORM, - cwd=_cwd(payload), - t=t, - data=data, - ) - - -def _print_permissive(event: str) -> None: - out = sys.__stdout__ or sys.stdout - try: - if event.startswith("before"): - out.write('{"permission": "allow"}') - else: - out.write('{"continue": true}') - out.flush() - except Exception: - pass - - -def _h_session_start(payload: dict) -> None: - _emit(payload, "session_start") - - -def _h_session_end(payload: dict) -> None: - sid = _session_id(payload) - if not sid: - return - config = Config.load() - store = Store(config) - seq = store.append_event( - session_id=sid, - platform=_PLATFORM, - cwd=_cwd(payload), - t="session_end", - data=_strip(payload), - ) - store.close_session(sid, platform=_PLATFORM) - capture_usage_cursor( - thirdeye_home=config.root, - session_id=sid, - payload=payload, - triggering_seq=seq if seq is not None else 0, - ) - - -def _h_before_submit(payload: dict) -> None: - sid = _session_id(payload) - if not sid: - return - config = Config.load() - seq = Store(config).append_event( - session_id=sid, - platform=_PLATFORM, - cwd=_cwd(payload), - t="user_message", - data=_strip(payload), - ) - try: - prompt = _get_str(payload, "prompt", "input", "text") - tags = extract_hashtags(prompt) - if not tags: - return - sd = session_dir(config.root, _PLATFORM, sid) - ts = TagStore(sd) - for tag in tags: - ts.add(seq, tag, source="auto") - mp = meta_path(sd) - m = read_meta(mp) - if m is not None: - m.tag_count = ts.tagged_seq_count() - write_meta(mp, m) - except Exception: - pass - - -def _h_after_response(payload: dict) -> None: - _emit(payload, "assistant_message") - - -def _h_before_shell(payload: dict) -> None: - _emit(payload, "tool_call", {"tool_name": "shell"}) - - -def _h_after_shell(payload: dict) -> None: - _emit(payload, "tool_result", {"tool_name": "shell"}) - - -def _h_before_mcp(payload: dict) -> None: - name = _get_str(payload, "tool_name", "toolName", "name", default="mcp") - _emit(payload, "tool_call", {"tool_name": name}) - - -def _h_after_mcp(payload: dict) -> None: - name = _get_str(payload, "tool_name", "toolName", "name", default="mcp") - _emit(payload, "tool_result", {"tool_name": name}) - - -def _h_before_read_file(payload: dict) -> None: - _emit(payload, "tool_call", {"tool_name": "read_file"}) - - -def _h_after_file_edit(payload: dict) -> None: - _emit(payload, "tool_result", {"tool_name": "edit_file"}) - - -def _h_before_tab_read(payload: dict) -> None: - _emit(payload, "tool_call", {"tool_name": "tab_read_file"}) - - -def _h_after_tab_edit(payload: dict) -> None: - _emit(payload, "tool_result", {"tool_name": "tab_edit_file"}) - - -def _h_post_tool_use(payload: dict) -> None: - name = _get_str(payload, "tool_name", "toolName", "name", "tool") - if name and name.lower() in DEDICATED_TOOL_NAMES: - return - if not name: - name = "unknown" - _emit(payload, "tool_result", {"tool_name": name}) - - -def _h_stop(payload: dict) -> None: - sid = _session_id(payload) - if not sid: - return - config = Config.load() - # TODO: Store does not expose a max_seq(session_id) helper. Pass 0 until - # such an API is added rather than introducing a new Store method here. - triggering_seq = 0 - capture_usage_cursor( - thirdeye_home=config.root, - session_id=sid, - payload=payload, - triggering_seq=triggering_seq, - ) - - -_HANDLERS: dict[str, Callable[[dict], None]] = { - "sessionStart": _h_session_start, - "sessionEnd": _h_session_end, - "beforeSubmitPrompt": _h_before_submit, - "afterAgentResponse": _h_after_response, - "beforeShellExecution": _h_before_shell, - "afterShellExecution": _h_after_shell, - "beforeMCPExecution": _h_before_mcp, - "afterMCPExecution": _h_after_mcp, - "beforeReadFile": _h_before_read_file, - "afterFileEdit": _h_after_file_edit, - "beforeTabFileRead": _h_before_tab_read, - "afterTabFileEdit": _h_after_tab_edit, - "postToolUse": _h_post_tool_use, - "stop": _h_stop, -} - - -def main() -> int: - event = "" - try: - payload = _read_stdin() - event = _event_name(payload) - handler = _HANDLERS.get(event) - if handler is not None: - handler(payload) - except Exception: - pass - finally: - _print_permissive(event) - return 0 diff --git a/src/thirdeye/platforms/cursor/install.py b/src/thirdeye/platforms/cursor/install.py deleted file mode 100644 index 9392499..0000000 --- a/src/thirdeye/platforms/cursor/install.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import annotations - -import json -import shutil -from pathlib import Path - -from thirdeye.platforms.base import Platform -from thirdeye.platforms.cursor.constants import ( - DISPLAY_NAME, - HOOK_BIN_NAME, - HOOK_TIMEOUT_S, - HOOKS_FILE, - PLATFORM_NAME, - TRACED_EVENTS, -) - - -def _load(path: Path) -> dict: - if not path.exists(): - return {"version": 1, "hooks": {}} - try: - data = json.loads(path.read_text()) - if not isinstance(data, dict): - return {"version": 1, "hooks": {}} - data.setdefault("version", 1) - data.setdefault("hooks", {}) - return data - except (json.JSONDecodeError, OSError): - return {"version": 1, "hooks": {}} - - -def _save(path: Path, data: dict) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(data, indent=2) + "\n") - - -def _resolve_command() -> str: - return shutil.which(HOOK_BIN_NAME) or HOOK_BIN_NAME - - -def _is_our_entry(entry: dict) -> bool: - cmd = entry.get("command", "") - return Path(cmd).name == HOOK_BIN_NAME - - -class CursorPlatform(Platform): - name = PLATFORM_NAME - display_name = DISPLAY_NAME - - def __init__(self, hooks_file: Path | None = None) -> None: - self._hooks_file = hooks_file or HOOKS_FILE - - def install(self) -> None: - data = _load(self._hooks_file) - hooks = data["hooks"] - cmd = _resolve_command() - for event in TRACED_EVENTS: - entries = hooks.setdefault(event, []) - if not any(_is_our_entry(e) for e in entries): - entries.append({"type": "command", "command": cmd, "timeout": HOOK_TIMEOUT_S}) - _save(self._hooks_file, data) - - def uninstall(self) -> None: - if not self._hooks_file.exists(): - return - data = _load(self._hooks_file) - hooks = data.get("hooks", {}) - for event in list(hooks.keys()): - entries = [e for e in hooks[event] if not _is_our_entry(e)] - if entries: - hooks[event] = entries - else: - del hooks[event] - _save(self._hooks_file, data) diff --git a/src/thirdeye/platforms/cursor/usage.py b/src/thirdeye/platforms/cursor/usage.py deleted file mode 100644 index ab12727..0000000 --- a/src/thirdeye/platforms/cursor/usage.py +++ /dev/null @@ -1,81 +0,0 @@ -from __future__ import annotations - -import time -from pathlib import Path -from typing import Any - -from thirdeye.paths import session_dir -from thirdeye.usage.errlog import safe_capture -from thirdeye.usage.store import UsageStore -from thirdeye.usage.types import UsageRow - - -def _get_str(payload: dict, *keys: str, default: str = "") -> str: - """Return the first non-empty string-coerced value among the keys.""" - for k in keys: - v = payload.get(k) - if v is not None and v != "": - return str(v) - return default - - -def _get_int(payload: dict, *keys: str) -> int | None: - """Return the first parseable int among the keys; None if none parse. - - Accepts numeric strings (including "0") but rejects "" and "--". - """ - for k in keys: - v = payload.get(k) - if v is None or v == "" or v == "--": - continue - try: - return int(v) - except (TypeError, ValueError): - continue - return None - - -@safe_capture(phase="extract_usage", platform="cursor") -def capture_usage_cursor( - *, - thirdeye_home: Path, - session_id: str, - payload: dict[str, Any], - triggering_seq: int, -) -> int: - """Append a UsageRow extracted from a Cursor stop/sessionEnd payload. - - Returns the number of rows appended (0 or 1). Wrapped in @safe_capture so - any exception is logged to usage-errors.jsonl and the function returns None - instead of raising. - """ - model = _get_str(payload, "model", "model_name") - input_tokens = _get_int(payload, "input_tokens", "inputTokens") - output_tokens = _get_int(payload, "output_tokens", "outputTokens") - - if not model or input_tokens is None or output_tokens is None: - return 0 - - sd = session_dir(thirdeye_home, "cursor", session_id) - store = UsageStore(sd) - state = store.read_state() - if state.get("last_seq", -1) >= 0: - return 0 - - row = UsageRow( - session_id=session_id, - seq=triggering_seq, - ts=_now_iso(), - platform="cursor", - model=model, - input_tokens=input_tokens, - output_tokens=output_tokens, - total_tokens=input_tokens + output_tokens, - ) - store.append([row]) - store.write_state(last_seq=triggering_seq) - return 1 - - -def _now_iso() -> str: - return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) diff --git a/src/thirdeye/platforms/gemini/__init__.py b/src/thirdeye/platforms/gemini/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/thirdeye/platforms/gemini/constants.py b/src/thirdeye/platforms/gemini/constants.py deleted file mode 100644 index 9d7f2e7..0000000 --- a/src/thirdeye/platforms/gemini/constants.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -PLATFORM_NAME = "gemini" -DISPLAY_NAME = "Gemini CLI" -SETTINGS_DIR = Path.home() / ".gemini" -SETTINGS_FILE = SETTINGS_DIR / "settings.json" - -# Friendly hook name written into settings.json — used by uninstall to identify -# entries to remove. -HOOK_NAME = "thirdeye-tracing" - -# Per-hook timeout in milliseconds (Gemini's own default is 60000; we use 30s -# so a wedged hook can't stall the user's CLI for too long). -HOOK_TIMEOUT_MS = 30000 - -# Map of Gemini hook event name -> CLI entry-point script name registered -# in pyproject.toml [project.scripts]. Order is preserved when writing -# settings.json. -HOOK_EVENTS: dict[str, str] = { - "SessionStart": "thirdeye-gemini-session-start", - "SessionEnd": "thirdeye-gemini-session-end", - "BeforeAgent": "thirdeye-gemini-before-agent", - "AfterAgent": "thirdeye-gemini-after-agent", - "BeforeModel": "thirdeye-gemini-before-model", - "AfterModel": "thirdeye-gemini-after-model", - "BeforeTool": "thirdeye-gemini-before-tool", - "AfterTool": "thirdeye-gemini-after-tool", -} diff --git a/src/thirdeye/platforms/gemini/hooks.py b/src/thirdeye/platforms/gemini/hooks.py deleted file mode 100644 index 8564d84..0000000 --- a/src/thirdeye/platforms/gemini/hooks.py +++ /dev/null @@ -1,209 +0,0 @@ -from __future__ import annotations - -import json -import os -import sys - -from thirdeye.config import Config -from thirdeye.env_capture import capture_env, env_to_tag -from thirdeye.meta import read_meta, write_meta -from thirdeye.paths import meta_path, session_dir -from thirdeye.store import Store -from thirdeye.tags import TagStore, extract_hashtags - -_PLATFORM = "gemini" - -# Routing keys we strip from event data because they're already used as -# routing fields when calling Store.append_event, OR because they're -# variants we don't want duplicated in storage. -_STRIP_KEYS = frozenset( - { - "session_id", - "sessionId", - "cwd", - "workingDir", - "working_dir", - } -) - - -def _read_stdin() -> dict: - try: - raw = sys.stdin.read() - except OSError: - return {} - if not raw: - return {} - try: - return json.loads(raw) - except json.JSONDecodeError: - return {} - - -def _flex_get(d: dict, *keys, default=None): - """Try multiple key names, return first non-None/non-empty value.""" - for key in keys: - v = d.get(key) - if v is not None and v != "": - return v - return default - - -def _strip_payload(payload: dict) -> dict: - return {k: v for k, v in payload.items() if k not in _STRIP_KEYS} - - -def _print_response() -> None: - """Gemini hooks must print {} to stdout when they finish.""" - print(json.dumps({})) - - -def _emit(t: str, payload: dict) -> int | None: - sid = _flex_get(payload, "session_id", "sessionId") - if not sid: - return None - cwd = _flex_get(payload, "cwd", "workingDir", "working_dir") or os.getcwd() - return Store(Config.load()).append_event( - session_id=sid, - platform=_PLATFORM, - cwd=cwd, - t=t, - data=_strip_payload(payload), - ) - - -def session_start() -> None: - try: - payload = _read_stdin() - sid = _flex_get(payload, "session_id", "sessionId") - seq = _emit("session_start", payload) - if seq is None: - return - config = Config.load() - captured = capture_env(config.capture_env_patterns) - if not captured: - return - sd = session_dir(config.root, _PLATFORM, sid) - tagstore = TagStore(sd) - for name, value in captured.items(): - tag = env_to_tag(name, value) - if tag is None: - continue - tagstore.add(seq, tag, source="auto") - except Exception: - pass - finally: - _print_response() - - -def session_end() -> None: - try: - payload = _read_stdin() - if _emit("session_end", payload) is not None: - sid = _flex_get(payload, "session_id", "sessionId") - Store(Config.load()).close_session(sid, platform=_PLATFORM) - except Exception: - pass - finally: - _print_response() - - -def before_agent() -> None: - try: - payload = _read_stdin() - sid = _flex_get(payload, "session_id", "sessionId") - if sid: - cwd = _flex_get(payload, "cwd", "workingDir", "working_dir") or os.getcwd() - config = Config.load() - seq = Store(config).append_event( - session_id=sid, - platform=_PLATFORM, - cwd=cwd, - t="user_message", - data=_strip_payload(payload), - ) - try: - prompt = _flex_get(payload, "prompt", "input", "userInput", "message", "user_input") - if isinstance(prompt, str) and prompt: - tags = extract_hashtags(prompt) - if tags: - sd = session_dir(config.root, _PLATFORM, sid) - tag_store = TagStore(sd) - for tag in sorted(tags): - tag_store.add(seq, tag, source="auto") - mp = meta_path(sd) - m = read_meta(mp) - if m is not None: - m.tag_count = tag_store.tagged_seq_count() - write_meta(mp, m) - except Exception: - pass - except Exception: - pass - finally: - _print_response() - - -def after_agent() -> None: - try: - _emit("assistant_message", _read_stdin()) - except Exception: - pass - finally: - _print_response() - - -def before_model() -> None: - try: - _emit("model_request", _read_stdin()) - except Exception: - pass - finally: - _print_response() - - -def after_model() -> None: - from thirdeye.platforms.gemini.usage import capture_usage_gemini - - try: - payload = _read_stdin() - sid = _flex_get(payload, "session_id", "sessionId") - if not sid: - return - cwd = _flex_get(payload, "cwd", "workingDir", "working_dir") or os.getcwd() - config = Config.load() - seq = Store(config).append_event( - session_id=sid, - platform=_PLATFORM, - cwd=cwd, - t="model_response", - data=_strip_payload(payload), - ) - capture_usage_gemini( - thirdeye_home=config.root, - session_id=sid, - payload=payload, - triggering_seq=seq, - ) - except Exception: - pass - finally: - _print_response() - - -def before_tool() -> None: - try: - _emit("tool_call", _read_stdin()) - except Exception: - pass - finally: - _print_response() - - -def after_tool() -> None: - try: - _emit("tool_result", _read_stdin()) - except Exception: - pass - finally: - _print_response() diff --git a/src/thirdeye/platforms/gemini/install.py b/src/thirdeye/platforms/gemini/install.py deleted file mode 100644 index 66e9bae..0000000 --- a/src/thirdeye/platforms/gemini/install.py +++ /dev/null @@ -1,101 +0,0 @@ -from __future__ import annotations - -import json -import shutil -from pathlib import Path - -from thirdeye.platforms.base import Platform -from thirdeye.platforms.gemini.constants import ( - DISPLAY_NAME, - HOOK_EVENTS, - HOOK_NAME, - HOOK_TIMEOUT_MS, - PLATFORM_NAME, - SETTINGS_FILE, -) - - -def _load(path: Path) -> dict: - if not path.exists(): - return {} - try: - return json.loads(path.read_text()) - except (json.JSONDecodeError, OSError): - return {} - - -def _save(path: Path, data: dict) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(data, indent=2) + "\n") - - -def _resolve_command(script_name: str) -> str: - return shutil.which(script_name) or script_name - - -class GeminiPlatform(Platform): - name = PLATFORM_NAME - display_name = DISPLAY_NAME - - def __init__(self, settings_file: Path | None = None) -> None: - self._settings_file = settings_file or SETTINGS_FILE - - def install(self) -> None: - settings = _load(self._settings_file) - hooks = settings.setdefault("hooks", {}) - - for event, script in HOOK_EVENTS.items(): - cmd = _resolve_command(script) - blocks: list = hooks.setdefault(event, []) - # Filter out any existing block that contains our hook name - blocks[:] = [ - block - for block in blocks - if not any(h.get("name") == HOOK_NAME for h in block.get("hooks", [])) - ] - # Append our block - blocks.append( - { - "matcher": "", - "hooks": [ - { - "type": "command", - "name": HOOK_NAME, - "command": cmd, - "timeout": HOOK_TIMEOUT_MS, - } - ], - } - ) - - _save(self._settings_file, settings) - - def uninstall(self) -> None: - if not self._settings_file.exists(): - return - - settings = _load(self._settings_file) - hooks = settings.get("hooks") - if not hooks: - if not settings: - self._settings_file.unlink(missing_ok=True) - return - - for event in list(hooks.keys()): - blocks = hooks[event] - blocks[:] = [ - block - for block in blocks - if not any(h.get("name") == HOOK_NAME for h in block.get("hooks", [])) - ] - if not blocks: - del hooks[event] - - if not hooks: - del settings["hooks"] - - if not settings: - self._settings_file.unlink(missing_ok=True) - return - - _save(self._settings_file, settings) diff --git a/src/thirdeye/platforms/gemini/usage.py b/src/thirdeye/platforms/gemini/usage.py deleted file mode 100644 index 3bf69df..0000000 --- a/src/thirdeye/platforms/gemini/usage.py +++ /dev/null @@ -1,56 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -from thirdeye.paths import session_dir -from thirdeye.usage.errlog import safe_capture -from thirdeye.usage.store import UsageStore -from thirdeye.usage.types import UsageRow - - -@safe_capture(phase="extract_usage", platform="gemini") -def capture_usage_gemini( - *, - thirdeye_home: Path, - session_id: str, - payload: dict, - triggering_seq: int, -) -> int: - """Build one UsageRow from a Gemini after_model payload, append it. - - Returns 1 if a row was appended, 0 if the payload had no usable usage - (e.g. an intermediate reasoning pass with empty usageMetadata). - """ - llm_response = payload.get("llm_response") if isinstance(payload, dict) else None - if not isinstance(llm_response, dict): - return 0 - usage_meta = llm_response.get("usageMetadata") - if not isinstance(usage_meta, dict): - return 0 - total = usage_meta.get("totalTokenCount") - if not total: - return 0 - - llm_request = payload.get("llm_request") if isinstance(payload, dict) else None - model = "" - if isinstance(llm_request, dict): - model = str(llm_request.get("model") or "") - if not model: - model = "unknown" - - row = UsageRow( - session_id=session_id, - seq=triggering_seq, - ts=str(payload.get("timestamp") or ""), - platform="gemini", - model=model, - input_tokens=int(usage_meta.get("promptTokenCount", 0)), - output_tokens=int(usage_meta.get("candidatesTokenCount", 0)), - total_tokens=int(total), - ) - - sd = session_dir(thirdeye_home, "gemini", session_id) - store = UsageStore(sd) - store.append([row]) - store.write_state(last_seq=triggering_seq) - return 1 diff --git a/src/thirdeye/timeparse.py b/src/thirdeye/timeparse.py index e439d82..fc4b5c9 100644 --- a/src/thirdeye/timeparse.py +++ b/src/thirdeye/timeparse.py @@ -6,7 +6,7 @@ _ROLLING_RE = re.compile(r"^(\d+)(d|h|m)$") _ISO_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") -_ERR = "could not parse {input!r}: " "expected ISO date, 'NN(d|h|m)', 'today', or 'yesterday'" +_ERR = "could not parse {input!r}: expected ISO date, 'NN(d|h|m)', 'today', or 'yesterday'" def parse_when(s: str | None, *, now: datetime | None = None) -> datetime | None: diff --git a/tests/fixtures/usage/gemini_model_response.json b/tests/fixtures/usage/gemini_model_response.json deleted file mode 100644 index 7cdd37b..0000000 --- a/tests/fixtures/usage/gemini_model_response.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "session_id": "127de361-c6ee-438e-9e05-6c31ee0047a6", - "timestamp": "2026-05-15T23:02:58.803Z", - "hook_event_name": "AfterModel", - "llm_request": { - "model": "gemini-3-flash-preview", - "messages": [{"role": "user", "content": "what's 2+2? answer in one word"}], - "config": {"temperature": 1, "topP": 0.95, "topK": 64} - }, - "llm_response": { - "text": "Four", - "candidates": [{"content": {"role": "model", "parts": ["Four"]}, "finishReason": "STOP"}], - "usageMetadata": { - "promptTokenCount": 9582, - "candidatesTokenCount": 1, - "totalTokenCount": 9748 - } - } -} diff --git a/tests/test_add_command.py b/tests/test_add_command.py index e19b0cb..7c040f2 100644 --- a/tests/test_add_command.py +++ b/tests/test_add_command.py @@ -6,11 +6,9 @@ from click.testing import CliRunner from thirdeye.cli import main -from thirdeye.commands.add import PLATFORMS +from thirdeye.commands.add import PLATFORMS, find_orphaned_hooks from thirdeye.platforms.claude.install import ClaudePlatform from thirdeye.platforms.codex.install import CodexPlatform -from thirdeye.platforms.cursor.install import CursorPlatform -from thirdeye.platforms.gemini.install import GeminiPlatform # -- command registration ------------------------------------------------------ @@ -39,6 +37,18 @@ def test_remove_help_mentions_claude(): assert "--claude" in r.output +def test_add_help_mentions_codex(): + r = CliRunner().invoke(main, ["add", "--help"]) + assert r.exit_code == 0 + assert "--codex" in r.output + + +def test_remove_help_mentions_codex(): + r = CliRunner().invoke(main, ["remove", "--help"]) + assert r.exit_code == 0 + assert "--codex" in r.output + + # -- platform flag required ---------------------------------------------------- @@ -202,202 +212,82 @@ def test_ingest_still_works(tmp_path: Path): # -- PLATFORMS dict correctness ------------------------------------------------ +def test_platforms_dict_is_exactly_claude_and_codex(): + assert set(PLATFORMS) == {"claude", "codex"} + + def test_platforms_dict_has_claude(): assert "claude" in PLATFORMS assert PLATFORMS["claude"] is ClaudePlatform -def test_platform_flag_value_maps_to_platforms_key(): - for key, cls in PLATFORMS.items(): - instance = cls() - assert instance.name == key - - -# -- implementation uses PLATFORMS dict ---------------------------------------- - - -def test_add_uses_platforms_dict(monkeypatch): - """The add command should dispatch via PLATFORMS[platform_flag](), not hardcode ClaudePlatform().""" - from unittest.mock import MagicMock - - mock_platform = MagicMock() - mock_platform.display_name = "Mock Platform" - mock_cls = MagicMock(return_value=mock_platform) - - monkeypatch.setitem(PLATFORMS, "claude", mock_cls) - r = CliRunner().invoke(main, ["add", "--claude"]) - assert r.exit_code == 0, r.output - mock_cls.assert_called_once() - mock_platform.install.assert_called_once() - - -def test_remove_uses_platforms_dict(monkeypatch): - """The remove command should dispatch via PLATFORMS[platform_flag](), not hardcode ClaudePlatform().""" - from unittest.mock import MagicMock - - mock_platform = MagicMock() - mock_platform.display_name = "Mock Platform" - mock_cls = MagicMock(return_value=mock_platform) - - monkeypatch.setitem(PLATFORMS, "claude", mock_cls) - r = CliRunner().invoke(main, ["remove", "--claude"]) - assert r.exit_code == 0, r.output - mock_cls.assert_called_once() - mock_platform.uninstall.assert_called_once() - - -# -- hook command structure ---------------------------------------------------- - - -def test_add_claude_hook_entries_have_command_type(tmp_path: Path, monkeypatch): - settings = tmp_path / "settings.json" - monkeypatch.setitem(PLATFORMS, "claude", lambda: ClaudePlatform(settings_file=settings)) - CliRunner().invoke(main, ["add", "--claude"]) - data = json.loads(settings.read_text()) - for event_name, entries in data["hooks"].items(): - for entry in entries: - for hook in entry["hooks"]: - assert hook["type"] == "command", f"{event_name} hook missing type=command" - assert "command" in hook, f"{event_name} hook missing command key" - - -def test_add_claude_hook_commands_contain_thirdeye(tmp_path: Path, monkeypatch): - settings = tmp_path / "settings.json" - monkeypatch.setitem(PLATFORMS, "claude", lambda: ClaudePlatform(settings_file=settings)) - CliRunner().invoke(main, ["add", "--claude"]) - data = json.loads(settings.read_text()) - for event_name, entries in data["hooks"].items(): - for entry in entries: - for hook in entry["hooks"]: - cmd = hook["command"] - assert "thirdeye" in cmd, f"{event_name} command {cmd!r} missing 'thirdeye'" - - -# -- PLATFORMS dict: gemini and codex ------------------------------------------ - - -def test_platforms_dict_has_gemini(): - assert "gemini" in PLATFORMS - assert PLATFORMS["gemini"] is GeminiPlatform - - def test_platforms_dict_has_codex(): assert "codex" in PLATFORMS assert PLATFORMS["codex"] is CodexPlatform -# -- help text lists all platform flags ---------------------------------------- - - -def test_add_help_mentions_gemini(): - r = CliRunner().invoke(main, ["add", "--help"]) - assert r.exit_code == 0 - assert "--gemini" in r.output - - -def test_add_help_mentions_codex(): - r = CliRunner().invoke(main, ["add", "--help"]) - assert r.exit_code == 0 - assert "--codex" in r.output +def test_platform_flag_value_maps_to_platforms_key(): + for key, cls in PLATFORMS.items(): + instance = cls() + assert instance.name == key -def test_remove_help_mentions_gemini(): - r = CliRunner().invoke(main, ["remove", "--help"]) - assert r.exit_code == 0 - assert "--gemini" in r.output +# -- removed platforms rejected ------------------------------------------------ -def test_remove_help_mentions_codex(): - r = CliRunner().invoke(main, ["remove", "--help"]) - assert r.exit_code == 0 - assert "--codex" in r.output +def test_add_gemini_fails(): + r = CliRunner().invoke(main, ["add", "gemini"]) + assert r.exit_code != 0 -# -- error message lists all three flags --------------------------------------- +def test_add_cursor_fails(): + r = CliRunner().invoke(main, ["add", "cursor"]) + assert r.exit_code != 0 -def test_add_no_flag_error_lists_all_platforms(): - r = CliRunner().invoke(main, ["add"]) +def test_add_gemini_flag_fails(): + r = CliRunner().invoke(main, ["add", "--gemini"]) assert r.exit_code != 0 - assert "--claude" in r.output - assert "--gemini" in r.output - assert "--codex" in r.output -def test_remove_no_flag_error_lists_all_platforms(): - r = CliRunner().invoke(main, ["remove"]) +def test_add_cursor_flag_fails(): + r = CliRunner().invoke(main, ["add", "--cursor"]) assert r.exit_code != 0 - assert "--claude" in r.output - assert "--gemini" in r.output - assert "--codex" in r.output -# -- install (add --gemini) ---------------------------------------------------- +# -- implementation uses PLATFORMS dict ---------------------------------------- -def test_add_gemini_calls_install(monkeypatch): +def test_add_uses_platforms_dict(monkeypatch): + """The add command should dispatch via PLATFORMS[platform_flag](), not hardcode ClaudePlatform().""" from unittest.mock import MagicMock mock_platform = MagicMock() - mock_platform.display_name = "Gemini CLI" + mock_platform.display_name = "Mock Platform" mock_cls = MagicMock(return_value=mock_platform) - monkeypatch.setitem(PLATFORMS, "gemini", mock_cls) - r = CliRunner().invoke(main, ["add", "--gemini"]) + monkeypatch.setitem(PLATFORMS, "claude", mock_cls) + r = CliRunner().invoke(main, ["add", "--claude"]) assert r.exit_code == 0, r.output mock_cls.assert_called_once() mock_platform.install.assert_called_once() -def test_add_gemini_writes_settings(tmp_path: Path, monkeypatch): - settings = tmp_path / "settings.json" - monkeypatch.setitem(PLATFORMS, "gemini", lambda: GeminiPlatform(settings_file=settings)) - r = CliRunner().invoke(main, ["add", "--gemini"]) - assert r.exit_code == 0, r.output - assert "Installed" in r.output - assert "Gemini CLI" in r.output - data = json.loads(settings.read_text()) - assert "hooks" in data - - -def test_add_gemini_registers_all_hook_events(tmp_path: Path, monkeypatch): - from thirdeye.platforms.gemini.constants import HOOK_EVENTS - - settings = tmp_path / "settings.json" - monkeypatch.setitem(PLATFORMS, "gemini", lambda: GeminiPlatform(settings_file=settings)) - CliRunner().invoke(main, ["add", "--gemini"]) - data = json.loads(settings.read_text()) - assert set(data["hooks"].keys()) == set(HOOK_EVENTS.keys()) - - -# -- uninstall (remove --gemini) ----------------------------------------------- - - -def test_remove_gemini_calls_uninstall(monkeypatch): +def test_remove_uses_platforms_dict(monkeypatch): + """The remove command should dispatch via PLATFORMS[platform_flag](), not hardcode ClaudePlatform().""" from unittest.mock import MagicMock mock_platform = MagicMock() - mock_platform.display_name = "Gemini CLI" + mock_platform.display_name = "Mock Platform" mock_cls = MagicMock(return_value=mock_platform) - monkeypatch.setitem(PLATFORMS, "gemini", mock_cls) - r = CliRunner().invoke(main, ["remove", "--gemini"]) + monkeypatch.setitem(PLATFORMS, "claude", mock_cls) + r = CliRunner().invoke(main, ["remove", "--claude"]) assert r.exit_code == 0, r.output mock_cls.assert_called_once() mock_platform.uninstall.assert_called_once() -def test_remove_gemini_removes_hooks(tmp_path: Path, monkeypatch): - settings = tmp_path / "settings.json" - monkeypatch.setitem(PLATFORMS, "gemini", lambda: GeminiPlatform(settings_file=settings)) - runner = CliRunner() - runner.invoke(main, ["add", "--gemini"]) - r = runner.invoke(main, ["remove", "--gemini"]) - assert r.exit_code == 0, r.output - assert "Removed" in r.output - assert "Gemini CLI" in r.output - - # -- install (add --codex) ----------------------------------------------------- @@ -427,9 +317,6 @@ def test_add_codex_writes_config(tmp_path: Path, monkeypatch): assert "thirdeye" in text -# -- uninstall (remove --codex) ------------------------------------------------ - - def test_remove_codex_calls_uninstall(monkeypatch): from unittest.mock import MagicMock @@ -455,103 +342,98 @@ def test_remove_codex_removes_notify(tmp_path: Path, monkeypatch): assert "Codex CLI" in r.output -# -- claude regression --------------------------------------------------------- - - -# -- cursor: help, dispatch, PLATFORMS dict ------------------------------------ - - -def test_add_help_mentions_cursor(): - r = CliRunner().invoke(main, ["add", "--help"]) - assert r.exit_code == 0 - assert "--cursor" in r.output - - -def test_remove_help_mentions_cursor(): - r = CliRunner().invoke(main, ["remove", "--help"]) - assert r.exit_code == 0 - assert "--cursor" in r.output - - -def test_platforms_dict_has_cursor(): - assert "cursor" in PLATFORMS - assert PLATFORMS["cursor"] is CursorPlatform - - -def test_add_cursor_dispatches_to_cursor_platform(monkeypatch): - from unittest.mock import MagicMock - - mock_platform = MagicMock() - mock_platform.display_name = "Cursor" - mock_cls = MagicMock(return_value=mock_platform) - - monkeypatch.setitem(PLATFORMS, "cursor", mock_cls) - r = CliRunner().invoke(main, ["add", "--cursor"]) - assert r.exit_code == 0, r.output - mock_cls.assert_called_once() - mock_platform.install.assert_called_once() - - -def test_remove_cursor_dispatches_to_cursor_platform(monkeypatch): - from unittest.mock import MagicMock +def test_codex_platform_name_matches_key(): + instance = CodexPlatform(config_file=Path("/fake")) + assert instance.name == "codex" - mock_platform = MagicMock() - mock_platform.display_name = "Cursor" - mock_cls = MagicMock(return_value=mock_platform) - monkeypatch.setitem(PLATFORMS, "cursor", mock_cls) - r = CliRunner().invoke(main, ["remove", "--cursor"]) - assert r.exit_code == 0, r.output - mock_cls.assert_called_once() - mock_platform.uninstall.assert_called_once() +# -- add --list ---------------------------------------------------------------- -def test_add_claude_still_works(tmp_path: Path, monkeypatch): - """Regression: --claude must still work after adding new platforms.""" - settings = tmp_path / "settings.json" - monkeypatch.setitem(PLATFORMS, "claude", lambda: ClaudePlatform(settings_file=settings)) - r = CliRunner().invoke(main, ["add", "--claude"]) +def test_list_shows_only_two_platforms(monkeypatch): + # Avoid depending on the real ~/.gemini or ~/.cursor config on this machine. + monkeypatch.setattr("thirdeye.commands.add.ORPHAN_CONFIG_PATHS", ()) + r = CliRunner().invoke(main, ["add", "--list"]) assert r.exit_code == 0, r.output - assert "Installed" in r.output - assert "Claude Code" in r.output - - -# -- platform flag_value maps to PLATFORMS key --------------------------------- - - -def test_all_platform_flag_values_map_to_platforms_keys(): - """Every key in PLATFORMS should have a corresponding --flag on the CLI.""" - for key in PLATFORMS: - r = CliRunner().invoke(main, ["add", f"--{key}", "--help"]) - # If the flag doesn't exist, Click will error before showing help - # We just need it not to fail with "no such option" - assert "no such option" not in r.output.lower(), f"--{key} flag not registered" - - -def test_gemini_platform_name_matches_key(): - instance = GeminiPlatform(settings_file=Path("/fake")) - assert instance.name == "gemini" + assert "claude" in r.output + assert "codex" in r.output + assert "gemini" not in r.output + assert "cursor" not in r.output + + +# -- find_orphaned_hooks ------------------------------------------------------- + + +def test_find_orphaned_hooks_detects_gemini_session_start(tmp_path: Path): + config = tmp_path / "settings.json" + config.write_text( + json.dumps( + { + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "/usr/local/bin/thirdeye-gemini-session-start", + "timeout": 30000, + } + ] + } + ] + } + } + ) + ) + result = find_orphaned_hooks([config]) + assert result == [(config, "/usr/local/bin/thirdeye-gemini-session-start")] + + +def test_find_orphaned_hooks_ignores_unrelated_commands(tmp_path: Path): + config = tmp_path / "settings.json" + config.write_text( + json.dumps( + { + "hooks": { + "SessionStart": [ + {"hooks": [{"type": "command", "command": "/usr/bin/echo hi"}]} + ] + } + } + ) + ) + assert find_orphaned_hooks([config]) == [] -def test_codex_platform_name_matches_key(): - instance = CodexPlatform(config_file=Path("/fake")) - assert instance.name == "codex" +def test_find_orphaned_hooks_missing_file(tmp_path: Path): + assert find_orphaned_hooks([tmp_path / "does-not-exist.json"]) == [] -# -- mutual exclusivity: last flag wins ---------------------------------------- +def test_find_orphaned_hooks_malformed_json(tmp_path: Path): + config = tmp_path / "settings.json" + config.write_text("{ this is not valid json ]") + assert find_orphaned_hooks([config]) == [] -def test_add_multiple_platform_flags_last_wins(monkeypatch): - """Passing two platform flags: last flag wins (Click flag_value behavior).""" - from unittest.mock import MagicMock +def test_find_orphaned_hooks_detects_cursor_hook_by_basename(tmp_path: Path): + config = tmp_path / "hooks.json" + config.write_text( + json.dumps( + { + "version": 1, + "hooks": { + "afterFileEdit": [{"command": "/opt/tools/thirdeye-cursor-hook", "timeout": 30}] + }, + } + ) + ) + result = find_orphaned_hooks([config]) + assert result == [(config, "/opt/tools/thirdeye-cursor-hook")] - mock_platform = MagicMock() - mock_platform.display_name = "Gemini CLI" - mock_cls = MagicMock(return_value=mock_platform) - monkeypatch.setitem(PLATFORMS, "gemini", mock_cls) - r = CliRunner().invoke(main, ["add", "--claude", "--gemini"]) - # Click's flag_value makes the last flag win, so gemini is resolved - assert r.exit_code == 0, r.output - mock_cls.assert_called_once() - mock_platform.install.assert_called_once() +def test_find_orphaned_hooks_does_not_match_claude_stop(tmp_path: Path): + config = tmp_path / "settings.json" + config.write_text( + json.dumps({"hooks": {"Stop": [{"hooks": [{"command": "thirdeye-claude-stop"}]}]}}) + ) + assert find_orphaned_hooks([config]) == [] diff --git a/tests/test_cursor_constants.py b/tests/test_cursor_constants.py deleted file mode 100644 index b73cfa5..0000000 --- a/tests/test_cursor_constants.py +++ /dev/null @@ -1,73 +0,0 @@ -from __future__ import annotations - -from thirdeye.platforms.cursor import constants - - -class TestNames: - def test_platform_name(self): - assert constants.PLATFORM_NAME == "cursor" - - def test_display_name(self): - assert constants.DISPLAY_NAME == "Cursor" - - def test_hook_bin_name(self): - assert constants.HOOK_BIN_NAME == "thirdeye-cursor-hook" - - -class TestHooksFile: - def test_under_cursor_dir(self): - assert constants.HOOKS_FILE.name == "hooks.json" - assert constants.HOOKS_FILE.parent.name == ".cursor" - - -class TestTracedEvents: - def test_contains_all_documented_events(self): - expected = { - "sessionStart", - "sessionEnd", - "beforeSubmitPrompt", - "afterAgentResponse", - "beforeShellExecution", - "afterShellExecution", - "beforeMCPExecution", - "afterMCPExecution", - "beforeReadFile", - "afterFileEdit", - "beforeTabFileRead", - "afterTabFileEdit", - "postToolUse", - "stop", - } - assert set(constants.TRACED_EVENTS) == expected - - def test_is_tuple_not_list(self): - assert isinstance(constants.TRACED_EVENTS, tuple) - - def test_no_duplicates(self): - assert len(constants.TRACED_EVENTS) == len(set(constants.TRACED_EVENTS)) - - -class TestDedicatedToolNames: - def test_includes_shell_variants(self): - for name in ("shell", "bash", "terminal", "run_command"): - assert name in constants.DEDICATED_TOOL_NAMES - - def test_includes_file_variants(self): - for name in ("read_file", "edit_file", "write_file"): - assert name in constants.DEDICATED_TOOL_NAMES - - def test_includes_mcp(self): - assert "mcp" in constants.DEDICATED_TOOL_NAMES - - -class TestStripKeys: - def test_strips_routing_keys(self): - for key in ("conversation_id", "conversationId", "hook_event_name", "hookEventName"): - assert key in constants.STRIP_KEYS - - def test_strips_pii(self): - assert "user_email" in constants.STRIP_KEYS - - def test_strips_noisy_paths(self): - assert "transcript_path" in constants.STRIP_KEYS - assert "workspace_roots" in constants.STRIP_KEYS diff --git a/tests/test_cursor_hook.py b/tests/test_cursor_hook.py deleted file mode 100644 index 9ca21d3..0000000 --- a/tests/test_cursor_hook.py +++ /dev/null @@ -1,578 +0,0 @@ -from __future__ import annotations - -import io -import json -from pathlib import Path - -import pytest - -from thirdeye.config import Config -from thirdeye.paths import session_dir, tags_path -from thirdeye.platforms.cursor import hook -from thirdeye.store import Store - - -@pytest.fixture -def env(monkeypatch, tmp_path: Path): - monkeypatch.setenv("THIRDEYE_HOME", str(tmp_path)) - return tmp_path - - -def _stdin(monkeypatch, payload: dict) -> None: - monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(payload))) - - -# -- _read_stdin --------------------------------------------------------------- - - -class TestReadStdin: - def test_valid_json(self, monkeypatch): - _stdin(monkeypatch, {"conversation_id": "abc", "cwd": "/p"}) - assert hook._read_stdin() == {"conversation_id": "abc", "cwd": "/p"} - - def test_empty_stdin_returns_empty_dict(self, monkeypatch): - monkeypatch.setattr("sys.stdin", io.StringIO("")) - assert hook._read_stdin() == {} - - def test_invalid_json_returns_empty_dict(self, monkeypatch): - monkeypatch.setattr("sys.stdin", io.StringIO("not json")) - assert hook._read_stdin() == {} - - def test_io_error_returns_empty_dict(self, monkeypatch): - class BrokenStdin: - def read(self): - raise OSError("broken pipe") - - monkeypatch.setattr("sys.stdin", BrokenStdin()) - assert hook._read_stdin() == {} - - -# -- payload helpers ----------------------------------------------------------- - - -class TestPayloadHelpers: - def test_event_name_reads_snake_case(self): - assert hook._event_name({"hook_event_name": "sessionStart"}) == "sessionStart" - - def test_event_name_reads_camel_case(self): - assert hook._event_name({"hookEventName": "sessionStart"}) == "sessionStart" - - def test_event_name_prefers_snake_when_both_present(self): - assert hook._event_name({"hook_event_name": "snake", "hookEventName": "camel"}) == "snake" - - def test_session_id_reads_both_cases(self): - assert hook._session_id({"conversation_id": "s1"}) == "s1" - assert hook._session_id({"conversationId": "s2"}) == "s2" - - def test_cwd_fallback_chain(self, monkeypatch, tmp_path: Path): - assert hook._cwd({"cwd": "/explicit"}) == "/explicit" - assert hook._cwd({"workspace_roots": ["/root/a", "/root/b"]}) == "/root/a" - monkeypatch.chdir(tmp_path) - assert hook._cwd({}) == str(tmp_path) - - def test_strip_removes_routing_and_pii_keys(self): - payload = { - "conversation_id": "abc", - "conversationId": "abc", - "workspace_roots": ["/r"], - "transcript_path": "/x", - "user_email": "u@example.com", - "hook_event_name": "sessionStart", - "hookEventName": "sessionStart", - "prompt": "hi", - } - assert hook._strip(payload) == {"prompt": "hi"} - - -# -- _print_permissive --------------------------------------------------------- - - -class TestPrintPermissive: - def test_before_events_get_permission_allow(self, capfd): - hook._print_permissive("beforeSubmitPrompt") - out = capfd.readouterr().out - assert '"permission": "allow"' in out - - def test_other_events_get_continue_true(self, capfd): - hook._print_permissive("afterAgentResponse") - out = capfd.readouterr().out - assert '"continue": true' in out - - def test_main_prints_permissive_even_when_handler_raises(self, monkeypatch, env: Path, capfd): - def boom(payload): - raise RuntimeError("nope") - - monkeypatch.setitem(hook._HANDLERS, "sessionStart", boom) - _stdin( - monkeypatch, - {"hook_event_name": "sessionStart", "conversation_id": "s1"}, - ) - rc = hook.main() - assert rc == 0 - out = capfd.readouterr().out - # sessionStart doesn't start with "before" → continue - assert '"continue": true' in out - - -# -- dispatch ------------------------------------------------------------------ - - -class TestDispatch: - def _events(self, sid: str) -> list[dict]: - return list(Store(Config.load()).reader(sid).iter_events()) - - def test_session_start(self, monkeypatch, env: Path): - _stdin( - monkeypatch, - { - "hook_event_name": "sessionStart", - "conversation_id": "s1", - "cwd": "/p", - "model": "composer-1", - }, - ) - hook.main() - events = self._events("s1") - assert len(events) == 1 - assert events[0]["t"] == "session_start" - assert events[0]["data"]["model"] == "composer-1" - assert "conversation_id" not in events[0]["data"] - assert "hook_event_name" not in events[0]["data"] - - def test_before_submit_prompt(self, monkeypatch, env: Path): - _stdin( - monkeypatch, - { - "hook_event_name": "beforeSubmitPrompt", - "conversation_id": "s1", - "cwd": "/p", - "prompt": "hello", - }, - ) - hook.main() - events = self._events("s1") - assert events[0]["t"] == "user_message" - assert events[0]["data"]["prompt"] == "hello" - - def test_after_agent_response(self, monkeypatch, env: Path): - _stdin( - monkeypatch, - { - "hook_event_name": "afterAgentResponse", - "conversation_id": "s1", - "cwd": "/p", - "response": "done", - }, - ) - hook.main() - events = self._events("s1") - assert events[0]["t"] == "assistant_message" - assert events[0]["data"]["response"] == "done" - - def test_before_shell(self, monkeypatch, env: Path): - _stdin( - monkeypatch, - { - "hook_event_name": "beforeShellExecution", - "conversation_id": "s1", - "cwd": "/p", - "command": "ls", - }, - ) - hook.main() - events = self._events("s1") - assert events[0]["t"] == "tool_call" - assert events[0]["data"]["tool_name"] == "shell" - assert events[0]["data"]["command"] == "ls" - - def test_after_shell(self, monkeypatch, env: Path): - _stdin( - monkeypatch, - { - "hook_event_name": "afterShellExecution", - "conversation_id": "s1", - "cwd": "/p", - "exit_code": 0, - }, - ) - hook.main() - events = self._events("s1") - assert events[0]["t"] == "tool_result" - assert events[0]["data"]["tool_name"] == "shell" - - def test_before_mcp(self, monkeypatch, env: Path): - _stdin( - monkeypatch, - { - "hook_event_name": "beforeMCPExecution", - "conversation_id": "s1", - "cwd": "/p", - "tool_name": "my_mcp_tool", - }, - ) - hook.main() - events = self._events("s1") - assert events[0]["t"] == "tool_call" - assert events[0]["data"]["tool_name"] == "my_mcp_tool" - - def test_before_mcp_default_name(self, monkeypatch, env: Path): - _stdin( - monkeypatch, - { - "hook_event_name": "beforeMCPExecution", - "conversation_id": "s1", - "cwd": "/p", - }, - ) - hook.main() - events = self._events("s1") - assert events[0]["data"]["tool_name"] == "mcp" - - def test_after_mcp(self, monkeypatch, env: Path): - _stdin( - monkeypatch, - { - "hook_event_name": "afterMCPExecution", - "conversation_id": "s1", - "cwd": "/p", - "toolName": "some_tool", - }, - ) - hook.main() - events = self._events("s1") - assert events[0]["t"] == "tool_result" - assert events[0]["data"]["tool_name"] == "some_tool" - - def test_before_read_file(self, monkeypatch, env: Path): - _stdin( - monkeypatch, - { - "hook_event_name": "beforeReadFile", - "conversation_id": "s1", - "cwd": "/p", - "file_path": "x.py", - }, - ) - hook.main() - events = self._events("s1") - assert events[0]["t"] == "tool_call" - assert events[0]["data"]["tool_name"] == "read_file" - - def test_after_file_edit(self, monkeypatch, env: Path): - _stdin( - monkeypatch, - { - "hook_event_name": "afterFileEdit", - "conversation_id": "s1", - "cwd": "/p", - "file_path": "x.py", - }, - ) - hook.main() - events = self._events("s1") - assert events[0]["t"] == "tool_result" - assert events[0]["data"]["tool_name"] == "edit_file" - - def test_before_tab_read(self, monkeypatch, env: Path): - _stdin( - monkeypatch, - { - "hook_event_name": "beforeTabFileRead", - "conversation_id": "s1", - "cwd": "/p", - }, - ) - hook.main() - events = self._events("s1") - assert events[0]["t"] == "tool_call" - assert events[0]["data"]["tool_name"] == "tab_read_file" - - def test_after_tab_edit(self, monkeypatch, env: Path): - _stdin( - monkeypatch, - { - "hook_event_name": "afterTabFileEdit", - "conversation_id": "s1", - "cwd": "/p", - }, - ) - hook.main() - events = self._events("s1") - assert events[0]["t"] == "tool_result" - assert events[0]["data"]["tool_name"] == "tab_edit_file" - - -# -- missing session id -------------------------------------------------------- - - -class TestMissingSessionId: - @pytest.mark.parametrize( - "event", - [ - "sessionStart", - "sessionEnd", - "beforeSubmitPrompt", - "afterAgentResponse", - "beforeShellExecution", - "afterShellExecution", - "beforeMCPExecution", - "afterMCPExecution", - "beforeReadFile", - "afterFileEdit", - "beforeTabFileRead", - "afterTabFileEdit", - "postToolUse", - "stop", - ], - ) - def test_no_event_appended(self, monkeypatch, env: Path, event: str): - _stdin(monkeypatch, {"hook_event_name": event, "cwd": "/p"}) - rc = hook.main() - assert rc == 0 - assert list(Store(Config.load()).list_sessions()) == [] - - -# -- beforeSubmitPrompt autotag ------------------------------------------------ - - -class TestBeforeSubmitAutotag: - def _tags_lines(self, env: Path, sid: str) -> list[dict]: - path = tags_path(session_dir(env, "cursor", sid)) - if not path.exists(): - return [] - return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] - - def test_hashtags_in_prompt_become_tags(self, monkeypatch, env: Path): - # Open the session first so meta exists. - _stdin( - monkeypatch, - {"hook_event_name": "sessionStart", "conversation_id": "s1", "cwd": "/p"}, - ) - hook.main() - - _stdin( - monkeypatch, - { - "hook_event_name": "beforeSubmitPrompt", - "conversation_id": "s1", - "cwd": "/p", - "prompt": "fix #bug in #parser", - }, - ) - hook.main() - - lines = self._tags_lines(env, "s1") - tags = {line["tag"] for line in lines} - assert tags == {"bug", "parser"} - for line in lines: - assert line["op"] == "add" - assert line["source"] == "auto" - - m = Store(Config.load()).get_meta("s1") - assert m.tag_count == 1 - - -# -- postToolUse dedup --------------------------------------------------------- - - -class TestPostToolUseDedup: - def _events(self, sid: str) -> list[dict]: - return list(Store(Config.load()).reader(sid).iter_events()) - - def test_skips_dedicated_tool_names(self, monkeypatch, env: Path): - _stdin( - monkeypatch, - { - "hook_event_name": "postToolUse", - "conversation_id": "s1", - "cwd": "/p", - "tool_name": "shell", - }, - ) - hook.main() - assert list(Store(Config.load()).list_sessions()) == [] - - def test_lowercases_for_comparison(self, monkeypatch, env: Path): - _stdin( - monkeypatch, - { - "hook_event_name": "postToolUse", - "conversation_id": "s1", - "cwd": "/p", - "tool_name": "Shell", - }, - ) - hook.main() - assert list(Store(Config.load()).list_sessions()) == [] - - def test_non_dedicated_tool_writes_event(self, monkeypatch, env: Path): - _stdin( - monkeypatch, - { - "hook_event_name": "postToolUse", - "conversation_id": "s1", - "cwd": "/p", - "tool_name": "my_custom_tool", - }, - ) - hook.main() - events = self._events("s1") - assert len(events) == 1 - assert events[0]["t"] == "tool_result" - assert events[0]["data"]["tool_name"] == "my_custom_tool" - - def test_unknown_tool_name_falls_back_to_unknown(self, monkeypatch, env: Path): - _stdin( - monkeypatch, - { - "hook_event_name": "postToolUse", - "conversation_id": "s1", - "cwd": "/p", - }, - ) - hook.main() - events = self._events("s1") - assert len(events) == 1 - assert events[0]["t"] == "tool_result" - assert events[0]["data"]["tool_name"] == "unknown" - - -# -- stop ---------------------------------------------------------------------- - - -class TestStop: - def test_stop_does_not_append_event(self, monkeypatch, env: Path): - _stdin( - monkeypatch, - { - "hook_event_name": "stop", - "conversation_id": "s1", - "cwd": "/p", - "model": "composer-1", - "input_tokens": 10, - "output_tokens": 5, - }, - ) - hook.main() - # No event was appended to the store. - assert list(Store(Config.load()).list_sessions()) == [] - - def test_stop_calls_capture_usage_cursor(self, monkeypatch, env: Path): - calls: list[dict] = [] - - def fake(*, thirdeye_home, session_id, payload, triggering_seq): - calls.append( - { - "thirdeye_home": thirdeye_home, - "session_id": session_id, - "payload": payload, - "triggering_seq": triggering_seq, - } - ) - return 1 - - monkeypatch.setattr(hook, "capture_usage_cursor", fake) - payload = { - "hook_event_name": "stop", - "conversation_id": "s1", - "cwd": "/p", - "model": "composer-1", - "input_tokens": 10, - "output_tokens": 5, - } - _stdin(monkeypatch, payload) - hook.main() - assert len(calls) == 1 - assert calls[0]["session_id"] == "s1" - assert calls[0]["payload"]["model"] == "composer-1" - assert "triggering_seq" in calls[0] - - def test_stop_with_missing_conversation_id_is_noop(self, monkeypatch, env: Path): - calls: list = [] - monkeypatch.setattr( - hook, - "capture_usage_cursor", - lambda **kw: calls.append(kw) or 0, - ) - _stdin(monkeypatch, {"hook_event_name": "stop", "cwd": "/p"}) - hook.main() - assert calls == [] - - -# -- sessionEnd ---------------------------------------------------------------- - - -class TestSessionEnd: - def test_appends_session_end_event(self, monkeypatch, env: Path): - _stdin( - monkeypatch, - {"hook_event_name": "sessionStart", "conversation_id": "s1", "cwd": "/p"}, - ) - hook.main() - _stdin( - monkeypatch, - {"hook_event_name": "sessionEnd", "conversation_id": "s1", "cwd": "/p"}, - ) - hook.main() - events = list(Store(Config.load()).reader("s1").iter_events()) - assert events[-1]["t"] == "session_end" - - def test_closes_session(self, monkeypatch, env: Path): - _stdin( - monkeypatch, - {"hook_event_name": "sessionStart", "conversation_id": "s1", "cwd": "/p"}, - ) - hook.main() - _stdin( - monkeypatch, - {"hook_event_name": "sessionEnd", "conversation_id": "s1", "cwd": "/p"}, - ) - hook.main() - m = next(Store(Config.load()).list_sessions()) - assert m.status == "closed" - assert m.ended_at is not None - - def test_invokes_usage_capture_as_fallback(self, monkeypatch, env: Path): - calls: list = [] - monkeypatch.setattr( - hook, - "capture_usage_cursor", - lambda **kw: calls.append(kw) or 0, - ) - _stdin( - monkeypatch, - {"hook_event_name": "sessionStart", "conversation_id": "s1", "cwd": "/p"}, - ) - hook.main() - _stdin( - monkeypatch, - { - "hook_event_name": "sessionEnd", - "conversation_id": "s1", - "cwd": "/p", - "model": "composer-1", - "input_tokens": 10, - "output_tokens": 5, - }, - ) - hook.main() - assert len(calls) == 1 - assert calls[0]["session_id"] == "s1" - # triggering_seq comes from the session_end seq which is >= 0. - assert calls[0]["triggering_seq"] >= 0 - - -# -- unknown event ------------------------------------------------------------- - - -class TestUnknownEvent: - def test_unknown_event_name_is_noop(self, monkeypatch, env: Path, capfd): - _stdin( - monkeypatch, - {"hook_event_name": "preCompact", "conversation_id": "s1", "cwd": "/p"}, - ) - rc = hook.main() - assert rc == 0 - assert list(Store(Config.load()).list_sessions()) == [] - out = capfd.readouterr().out - assert '"continue": true' in out diff --git a/tests/test_cursor_install.py b/tests/test_cursor_install.py deleted file mode 100644 index b3925fd..0000000 --- a/tests/test_cursor_install.py +++ /dev/null @@ -1,217 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -from thirdeye.platforms.cursor.constants import ( - HOOK_BIN_NAME, - HOOK_TIMEOUT_S, - TRACED_EVENTS, -) -from thirdeye.platforms.cursor.install import CursorPlatform - - -class TestCursorPlatformAttributes: - def test_name_is_cursor(self): - p = CursorPlatform(hooks_file=Path("/fake/hooks.json")) - assert p.name == "cursor" - - def test_display_name(self): - p = CursorPlatform(hooks_file=Path("/fake/hooks.json")) - assert p.display_name == "Cursor" - - def test_is_platform_subclass(self): - from thirdeye.platforms.base import Platform - - assert issubclass(CursorPlatform, Platform) - - -class TestInstallFreshFile: - def test_writes_all_traced_events(self, tmp_path: Path): - hooks_file = tmp_path / "hooks.json" - CursorPlatform(hooks_file=hooks_file).install() - data = json.loads(hooks_file.read_text()) - assert set(data["hooks"].keys()) == set(TRACED_EVENTS) - - def test_writes_version_1(self, tmp_path: Path): - hooks_file = tmp_path / "hooks.json" - CursorPlatform(hooks_file=hooks_file).install() - data = json.loads(hooks_file.read_text()) - assert data["version"] == 1 - - def test_creates_parent_dir(self, tmp_path: Path): - hooks_file = tmp_path / "nested" / "deeper" / "hooks.json" - CursorPlatform(hooks_file=hooks_file).install() - assert hooks_file.exists() - - def test_each_event_has_one_entry(self, tmp_path: Path): - hooks_file = tmp_path / "hooks.json" - CursorPlatform(hooks_file=hooks_file).install() - data = json.loads(hooks_file.read_text()) - for event, entries in data["hooks"].items(): - assert len(entries) == 1, f"expected 1 entry for {event}" - - def test_command_basename_matches_hook_bin_name(self, tmp_path: Path): - hooks_file = tmp_path / "hooks.json" - CursorPlatform(hooks_file=hooks_file).install() - data = json.loads(hooks_file.read_text()) - for event, entries in data["hooks"].items(): - for entry in entries: - assert Path(entry["command"]).name == HOOK_BIN_NAME - - def test_each_entry_has_command_type_and_timeout(self, tmp_path: Path): - hooks_file = tmp_path / "hooks.json" - CursorPlatform(hooks_file=hooks_file).install() - data = json.loads(hooks_file.read_text()) - for event, entries in data["hooks"].items(): - for entry in entries: - assert entry["type"] == "command" - assert entry["timeout"] == HOOK_TIMEOUT_S - assert isinstance(entry["command"], str) - - -class TestInstallIdempotent: - def test_running_install_twice_does_not_duplicate_entries(self, tmp_path: Path): - hooks_file = tmp_path / "hooks.json" - p = CursorPlatform(hooks_file=hooks_file) - p.install() - p.install() - data = json.loads(hooks_file.read_text()) - for event, entries in data["hooks"].items(): - ours = [e for e in entries if Path(e.get("command", "")).name == HOOK_BIN_NAME] - assert len(ours) == 1, f"expected 1 thirdeye entry for {event}" - - def test_running_install_after_path_change_dedups_by_basename( - self, tmp_path: Path, monkeypatch - ): - hooks_file = tmp_path / "hooks.json" - hooks_file.parent.mkdir(parents=True, exist_ok=True) - hooks_file.write_text( - json.dumps( - { - "version": 1, - "hooks": { - "sessionStart": [ - { - "type": "command", - "command": f"/old/path/{HOOK_BIN_NAME}", - "timeout": HOOK_TIMEOUT_S, - } - ] - }, - } - ) - ) - monkeypatch.setattr( - "thirdeye.platforms.cursor.install.shutil.which", - lambda _: f"/new/path/{HOOK_BIN_NAME}", - ) - CursorPlatform(hooks_file=hooks_file).install() - data = json.loads(hooks_file.read_text()) - ours = [ - e - for e in data["hooks"]["sessionStart"] - if Path(e.get("command", "")).name == HOOK_BIN_NAME - ] - assert len(ours) == 1 - - -class TestInstallPreservesExistingHooks: - def test_preserves_unrelated_hook_under_same_event(self, tmp_path: Path): - hooks_file = tmp_path / "hooks.json" - hooks_file.write_text( - json.dumps( - { - "version": 1, - "hooks": { - "sessionStart": [ - { - "type": "command", - "command": "/path/to/arize-hook-cursor", - "timeout": 30, - } - ] - }, - } - ) - ) - CursorPlatform(hooks_file=hooks_file).install() - data = json.loads(hooks_file.read_text()) - cmds = [e["command"] for e in data["hooks"]["sessionStart"]] - assert "/path/to/arize-hook-cursor" in cmds - assert any(Path(c).name == HOOK_BIN_NAME for c in cmds) - - def test_preserves_unknown_top_level_keys(self, tmp_path: Path): - hooks_file = tmp_path / "hooks.json" - hooks_file.write_text(json.dumps({"version": 1, "hooks": {}, "custom": "x"})) - CursorPlatform(hooks_file=hooks_file).install() - data = json.loads(hooks_file.read_text()) - assert data["custom"] == "x" - - -class TestUninstallFreshState: - def test_uninstall_missing_file_is_noop(self, tmp_path: Path): - hooks_file = tmp_path / "hooks.json" - CursorPlatform(hooks_file=hooks_file).uninstall() - assert not hooks_file.exists() - - def test_uninstall_empty_hooks_dict_is_noop(self, tmp_path: Path): - hooks_file = tmp_path / "hooks.json" - hooks_file.write_text(json.dumps({"version": 1, "hooks": {}})) - CursorPlatform(hooks_file=hooks_file).uninstall() - data = json.loads(hooks_file.read_text()) - assert data == {"version": 1, "hooks": {}} - - -class TestUninstallRemovesOnlyOurEntries: - def test_removes_only_thirdeye_entries(self, tmp_path: Path): - hooks_file = tmp_path / "hooks.json" - hooks_file.write_text( - json.dumps( - { - "version": 1, - "hooks": { - "sessionStart": [ - { - "type": "command", - "command": f"/usr/local/bin/{HOOK_BIN_NAME}", - "timeout": HOOK_TIMEOUT_S, - }, - { - "type": "command", - "command": "/path/to/arize-hook-cursor", - "timeout": 30, - }, - ] - }, - } - ) - ) - CursorPlatform(hooks_file=hooks_file).uninstall() - data = json.loads(hooks_file.read_text()) - cmds = [e["command"] for e in data["hooks"]["sessionStart"]] - assert "/path/to/arize-hook-cursor" in cmds - assert not any(Path(c).name == HOOK_BIN_NAME for c in cmds) - - def test_collapses_empty_event_arrays(self, tmp_path: Path): - hooks_file = tmp_path / "hooks.json" - p = CursorPlatform(hooks_file=hooks_file) - p.install() - p.uninstall() - data = json.loads(hooks_file.read_text()) - assert data["hooks"] == {} - - def test_preserves_version_field(self, tmp_path: Path): - hooks_file = tmp_path / "hooks.json" - p = CursorPlatform(hooks_file=hooks_file) - p.install() - p.uninstall() - data = json.loads(hooks_file.read_text()) - assert data["version"] == 1 - - def test_does_not_delete_file(self, tmp_path: Path): - hooks_file = tmp_path / "hooks.json" - p = CursorPlatform(hooks_file=hooks_file) - p.install() - p.uninstall() - assert hooks_file.exists() diff --git a/tests/test_cursor_usage.py b/tests/test_cursor_usage.py deleted file mode 100644 index 2161715..0000000 --- a/tests/test_cursor_usage.py +++ /dev/null @@ -1,245 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - -import pytest - -from thirdeye.paths import session_dir, usage_log_path -from thirdeye.platforms.cursor.usage import ( - _get_int, - _get_str, - capture_usage_cursor, -) -from thirdeye.usage.store import UsageStore - -SESSION_ID = "test-conv-123" - - -def _store_for(tmp_path: Path) -> UsageStore: - return UsageStore(session_dir(tmp_path, "cursor", SESSION_ID)) - - -def _rows(tmp_path: Path) -> list: - return list(_store_for(tmp_path).iter_rows()) - - -def _complete_snake_payload(**overrides: Any) -> dict[str, Any]: - payload = { - "model": "composer-1", - "input_tokens": 100, - "output_tokens": 50, - } - payload.update(overrides) - return payload - - -def _complete_camel_payload(**overrides: Any) -> dict[str, Any]: - payload = { - "model": "composer-1", - "inputTokens": 100, - "outputTokens": 50, - } - payload.update(overrides) - return payload - - -class TestGetStr: - def test_first_key_present_wins(self) -> None: - assert _get_str({"model": "a", "model_name": "b"}, "model", "model_name") == "a" - - def test_falls_back_to_second_key(self) -> None: - assert _get_str({"model_name": "b"}, "model", "model_name") == "b" - - def test_empty_string_falls_through(self) -> None: - assert _get_str({"model": "", "model_name": "b"}, "model", "model_name") == "b" - - def test_returns_default_when_all_missing(self) -> None: - assert _get_str({}, "model", "model_name", default="x") == "x" - assert _get_str({}, "model", "model_name") == "" - - -class TestGetInt: - def test_int_value(self) -> None: - assert _get_int({"input_tokens": 5}, "input_tokens") == 5 - - def test_string_int_value(self) -> None: - assert _get_int({"input_tokens": "42"}, "input_tokens") == 42 - - def test_zero_is_valid(self) -> None: - assert _get_int({"input_tokens": 0}, "input_tokens") == 0 - assert _get_int({"input_tokens": "0"}, "input_tokens") == 0 - - def test_empty_string_falls_through(self) -> None: - assert _get_int({"input_tokens": "", "inputTokens": 7}, "input_tokens", "inputTokens") == 7 - - def test_dash_dash_falls_through(self) -> None: - assert ( - _get_int({"input_tokens": "--", "inputTokens": 9}, "input_tokens", "inputTokens") == 9 - ) - - def test_returns_none_when_all_missing(self) -> None: - assert _get_int({}, "input_tokens", "inputTokens") is None - - def test_unparseable_falls_through_then_returns_none(self) -> None: - assert _get_int({"input_tokens": "abc"}, "input_tokens") is None - assert ( - _get_int({"input_tokens": "abc", "inputTokens": "xyz"}, "input_tokens", "inputTokens") - is None - ) - - -class TestCaptureFromCompletePayload: - def test_writes_one_row_with_correct_fields(self, tmp_path: Path) -> None: - result = capture_usage_cursor( - thirdeye_home=tmp_path, - session_id=SESSION_ID, - payload=_complete_snake_payload(), - triggering_seq=7, - ) - assert result == 1 - rows = _rows(tmp_path) - assert len(rows) == 1 - row = rows[0] - assert row.session_id == SESSION_ID - assert row.model == "composer-1" - assert row.input_tokens == 100 - assert row.output_tokens == 50 - assert row.total_tokens == 150 - - def test_camelcase_payload_works(self, tmp_path: Path) -> None: - result = capture_usage_cursor( - thirdeye_home=tmp_path, - session_id=SESSION_ID, - payload=_complete_camel_payload(), - triggering_seq=3, - ) - assert result == 1 - rows = _rows(tmp_path) - assert len(rows) == 1 - assert rows[0].input_tokens == 100 - assert rows[0].output_tokens == 50 - assert rows[0].total_tokens == 150 - - def test_zero_tokens_writes_row(self, tmp_path: Path) -> None: - result = capture_usage_cursor( - thirdeye_home=tmp_path, - session_id=SESSION_ID, - payload=_complete_snake_payload(input_tokens=0, output_tokens=0), - triggering_seq=1, - ) - assert result == 1 - rows = _rows(tmp_path) - assert len(rows) == 1 - assert rows[0].input_tokens == 0 - assert rows[0].output_tokens == 0 - assert rows[0].total_tokens == 0 - - def test_uses_triggering_seq(self, tmp_path: Path) -> None: - capture_usage_cursor( - thirdeye_home=tmp_path, - session_id=SESSION_ID, - payload=_complete_snake_payload(), - triggering_seq=42, - ) - rows = _rows(tmp_path) - assert rows[0].seq == 42 - - def test_platform_is_cursor(self, tmp_path: Path) -> None: - capture_usage_cursor( - thirdeye_home=tmp_path, - session_id=SESSION_ID, - payload=_complete_snake_payload(), - triggering_seq=1, - ) - rows = _rows(tmp_path) - assert rows[0].platform == "cursor" - - -class TestCaptureSkipsIncomplete: - def test_missing_model_returns_zero(self, tmp_path: Path) -> None: - payload = _complete_snake_payload() - payload.pop("model") - result = capture_usage_cursor( - thirdeye_home=tmp_path, - session_id=SESSION_ID, - payload=payload, - triggering_seq=1, - ) - assert result == 0 - - def test_missing_input_tokens_returns_zero(self, tmp_path: Path) -> None: - payload = _complete_snake_payload() - payload.pop("input_tokens") - result = capture_usage_cursor( - thirdeye_home=tmp_path, - session_id=SESSION_ID, - payload=payload, - triggering_seq=1, - ) - assert result == 0 - - def test_missing_output_tokens_returns_zero(self, tmp_path: Path) -> None: - payload = _complete_snake_payload() - payload.pop("output_tokens") - result = capture_usage_cursor( - thirdeye_home=tmp_path, - session_id=SESSION_ID, - payload=payload, - triggering_seq=1, - ) - assert result == 0 - - def test_returns_zero_writes_no_row(self, tmp_path: Path) -> None: - capture_usage_cursor( - thirdeye_home=tmp_path, - session_id=SESSION_ID, - payload={"input_tokens": 10, "output_tokens": 5}, - triggering_seq=1, - ) - assert _rows(tmp_path) == [] - - -class TestCaptureIdempotent: - def test_second_call_does_not_write_second_row(self, tmp_path: Path) -> None: - capture_usage_cursor( - thirdeye_home=tmp_path, - session_id=SESSION_ID, - payload=_complete_snake_payload(), - triggering_seq=1, - ) - capture_usage_cursor( - thirdeye_home=tmp_path, - session_id=SESSION_ID, - payload=_complete_snake_payload(), - triggering_seq=2, - ) - rows = _rows(tmp_path) - assert len(rows) == 1 - - -class TestCaptureErrorsAreSwallowed: - def test_broken_store_logs_error_and_returns_none( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - def boom(self, rows): - raise RuntimeError("disk full") - - monkeypatch.setattr(UsageStore, "append", boom) - - result = capture_usage_cursor( - thirdeye_home=tmp_path, - session_id=SESSION_ID, - payload=_complete_snake_payload(), - triggering_seq=1, - ) - assert result is None - - log = usage_log_path(tmp_path) - assert log.exists() - entry = json.loads(log.read_text().strip().splitlines()[-1]) - assert entry["platform"] == "cursor" - assert entry["phase"] == "extract_usage" - assert entry["session_id"] == SESSION_ID - assert entry["error_class"] == "RuntimeError" diff --git a/tests/test_e2e_gemini.py b/tests/test_e2e_gemini.py deleted file mode 100644 index 4594ea5..0000000 --- a/tests/test_e2e_gemini.py +++ /dev/null @@ -1,298 +0,0 @@ -from __future__ import annotations - -import io -import json -from pathlib import Path - -import pytest - -from thirdeye.config import Config -from thirdeye.platforms.gemini import hooks as g_hooks -from thirdeye.platforms.gemini.install import GeminiPlatform -from thirdeye.store import Store - - -@pytest.fixture -def env(monkeypatch, tmp_path: Path): - monkeypatch.setenv("THIRDEYE_HOME", str(tmp_path)) - return tmp_path - - -def _stdin(monkeypatch, payload: dict) -> None: - monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(payload))) - - -# -- install ------------------------------------------------------------------- - - -class TestGeminiInstall: - def test_install_writes_settings_file(self, tmp_path: Path): - settings_file = tmp_path / ".gemini" / "settings.json" - GeminiPlatform(settings_file=settings_file).install() - assert settings_file.exists() - - def test_install_writes_all_eight_events(self, tmp_path: Path): - settings_file = tmp_path / ".gemini" / "settings.json" - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - expected_events = { - "SessionStart", - "SessionEnd", - "BeforeAgent", - "AfterAgent", - "BeforeModel", - "AfterModel", - "BeforeTool", - "AfterTool", - } - assert set(settings["hooks"].keys()) == expected_events - - def test_install_each_hook_has_command_type(self, tmp_path: Path): - settings_file = tmp_path / ".gemini" / "settings.json" - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - for event, blocks in settings["hooks"].items(): - for block in blocks: - for h in block["hooks"]: - assert h["type"] == "command", f"{event} hook type != command" - assert "command" in h - - def test_install_each_hook_has_thirdeye_gemini_command(self, tmp_path: Path): - settings_file = tmp_path / ".gemini" / "settings.json" - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - for event, blocks in settings["hooks"].items(): - cmds = [h["command"] for block in blocks for h in block["hooks"]] - assert any( - "thirdeye-gemini" in c for c in cmds - ), f"no thirdeye-gemini command for {event}" - - def test_install_idempotent(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - p = GeminiPlatform(settings_file=settings_file) - p.install() - first = settings_file.read_text() - p.install() - second = settings_file.read_text() - assert first == second - - def test_install_output_is_valid_json(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - GeminiPlatform(settings_file=settings_file).install() - json.loads(settings_file.read_text()) # should not raise - - -# -- full lifecycle ------------------------------------------------------------ - - -class TestGeminiFullLifecycle: - def test_full_session_lifecycle(self, monkeypatch, env: Path): - """Drive all 8 hooks in order and verify events are stored correctly.""" - sid = "gemini-e2e-001" - base = {"session_id": sid, "cwd": "/proj/gemini"} - - # SessionStart - _stdin(monkeypatch, {**base, "source": "cli"}) - g_hooks.session_start() - - # BeforeAgent - _stdin(monkeypatch, {**base, "input": "explain this"}) - g_hooks.before_agent() - - # BeforeModel - _stdin(monkeypatch, {**base, "model": "gemini-pro"}) - g_hooks.before_model() - - # AfterModel - _stdin(monkeypatch, {**base, "response": "Here is the explanation"}) - g_hooks.after_model() - - # BeforeTool - _stdin(monkeypatch, {**base, "tool_name": "search", "query": "test"}) - g_hooks.before_tool() - - # AfterTool - _stdin(monkeypatch, {**base, "tool_name": "search", "result": "found it"}) - g_hooks.after_tool() - - # AfterAgent - _stdin(monkeypatch, {**base, "output": "done"}) - g_hooks.after_agent() - - # SessionEnd - _stdin(monkeypatch, base) - g_hooks.session_end() - - store = Store(Config.load()) - events = list(store.reader(sid).iter_events()) - types = [e["t"] for e in events] - assert types == [ - "session_start", - "user_message", - "model_request", - "model_response", - "tool_call", - "tool_result", - "assistant_message", - "session_end", - ] - - def test_session_end_closes_session(self, monkeypatch, env: Path): - sid = "gemini-close-001" - base = {"session_id": sid, "cwd": "/proj/x"} - - _stdin(monkeypatch, base) - g_hooks.session_start() - _stdin(monkeypatch, base) - g_hooks.session_end() - - store = Store(Config.load()) - m = next(store.list_sessions()) - assert m.status == "closed" - assert m.ended_at is not None - - def test_platform_is_gemini(self, monkeypatch, env: Path): - sid = "gemini-plat-001" - _stdin(monkeypatch, {"session_id": sid, "cwd": "/p"}) - g_hooks.session_start() - - m = next(Store(Config.load()).list_sessions()) - assert m.platform == "gemini" - - def test_event_count_matches(self, monkeypatch, env: Path): - sid = "gemini-count-001" - base = {"session_id": sid, "cwd": "/p"} - - _stdin(monkeypatch, base) - g_hooks.session_start() - _stdin(monkeypatch, {**base, "input": "hello"}) - g_hooks.before_agent() - _stdin(monkeypatch, base) - g_hooks.session_end() - - m = next(Store(Config.load()).list_sessions()) - assert m.event_count == 3 - - -# -- hooks print {} to stdout ------------------------------------------------- - - -class TestGeminiHooksPrintEmptyJson: - def test_session_start_prints_empty_json(self, monkeypatch, env: Path, capsys): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - g_hooks.session_start() - captured = capsys.readouterr() - assert captured.out.strip() == "{}" - - def test_session_end_prints_empty_json(self, monkeypatch, env: Path, capsys): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - g_hooks.session_start() - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - g_hooks.session_end() - captured = capsys.readouterr() - # Both session_start and session_end print {} - lines = [line for line in captured.out.strip().splitlines() if line.strip()] - assert all(line.strip() == "{}" for line in lines) - - def test_before_agent_prints_empty_json(self, monkeypatch, env: Path, capsys): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - g_hooks.before_agent() - captured = capsys.readouterr() - assert captured.out.strip() == "{}" - - def test_after_agent_prints_empty_json(self, monkeypatch, env: Path, capsys): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - g_hooks.after_agent() - captured = capsys.readouterr() - assert captured.out.strip() == "{}" - - def test_before_model_prints_empty_json(self, monkeypatch, env: Path, capsys): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - g_hooks.before_model() - captured = capsys.readouterr() - assert captured.out.strip() == "{}" - - def test_after_model_prints_empty_json(self, monkeypatch, env: Path, capsys): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - g_hooks.after_model() - captured = capsys.readouterr() - assert captured.out.strip() == "{}" - - def test_before_tool_prints_empty_json(self, monkeypatch, env: Path, capsys): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - g_hooks.before_tool() - captured = capsys.readouterr() - assert captured.out.strip() == "{}" - - def test_after_tool_prints_empty_json(self, monkeypatch, env: Path, capsys): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - g_hooks.after_tool() - captured = capsys.readouterr() - assert captured.out.strip() == "{}" - - def test_noop_hook_still_prints_empty_json(self, monkeypatch, env: Path, capsys): - """Even when session_id is missing, Gemini hooks must print {}.""" - _stdin(monkeypatch, {"cwd": "/p"}) - g_hooks.session_start() - captured = capsys.readouterr() - assert captured.out.strip() == "{}" - - -# -- flexible key lookup ------------------------------------------------------- - - -class TestGeminiFlexibleKeys: - def test_camel_case_session_id(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"sessionId": "camel-001", "cwd": "/p"}) - g_hooks.session_start() - events = list(Store(Config.load()).reader("camel-001").iter_events()) - assert len(events) == 1 - - def test_camel_case_working_dir(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "wd-001", "workingDir": "/my/dir"}) - g_hooks.session_start() - m = next(Store(Config.load()).list_sessions()) - assert m.cwd == "/my/dir" - - -# -- strip keys ---------------------------------------------------------------- - - -class TestGeminiStripKeys: - def test_session_id_stripped_from_data(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p", "extra": "val"}) - g_hooks.session_start() - events = list(Store(Config.load()).reader("s1").iter_events()) - data = events[0].get("data", {}) - assert "session_id" not in data - assert "cwd" not in data - assert data["extra"] == "val" - - -# -- silent noop edge cases ---------------------------------------------------- - - -class TestGeminiSilentNoop: - def test_missing_session_id_is_noop(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"cwd": "/p"}) - g_hooks.before_agent() - assert list(Store(Config.load()).list_sessions()) == [] - - def test_empty_stdin_is_noop(self, monkeypatch, env: Path): - monkeypatch.setattr("sys.stdin", io.StringIO("")) - g_hooks.session_start() - assert list(Store(Config.load()).list_sessions()) == [] - - def test_invalid_json_is_noop(self, monkeypatch, env: Path): - monkeypatch.setattr("sys.stdin", io.StringIO("not json")) - g_hooks.before_tool() - assert list(Store(Config.load()).list_sessions()) == [] - - def test_broken_stdin_is_noop(self, monkeypatch, env: Path): - class BrokenStdin: - def read(self): - raise OSError("broken pipe") - - monkeypatch.setattr("sys.stdin", BrokenStdin()) - g_hooks.after_model() - assert list(Store(Config.load()).list_sessions()) == [] diff --git a/tests/test_gemini_constants.py b/tests/test_gemini_constants.py deleted file mode 100644 index d30859d..0000000 --- a/tests/test_gemini_constants.py +++ /dev/null @@ -1,64 +0,0 @@ -from __future__ import annotations - -from thirdeye.platforms.gemini.constants import ( - DISPLAY_NAME, - HOOK_EVENTS, - HOOK_NAME, - HOOK_TIMEOUT_MS, - PLATFORM_NAME, - SETTINGS_DIR, - SETTINGS_FILE, -) - - -def test_platform_name(): - assert PLATFORM_NAME == "gemini" - - -def test_display_name(): - assert DISPLAY_NAME == "Gemini CLI" - - -def test_settings_file_under_gemini_home(): - parts = SETTINGS_FILE.parts - assert ".gemini" in parts - assert parts[-1] == "settings.json" - - -def test_settings_dir_is_parent_of_settings_file(): - assert SETTINGS_FILE.parent == SETTINGS_DIR - - -def test_hook_events_has_exactly_eight_entries(): - assert len(HOOK_EVENTS) == 8 - - -def test_hook_events_covers_known_lifecycle(): - expected = { - "SessionStart", - "SessionEnd", - "BeforeAgent", - "AfterAgent", - "BeforeModel", - "AfterModel", - "BeforeTool", - "AfterTool", - } - assert set(HOOK_EVENTS.keys()) == expected - - -def test_hook_event_scripts_unique(): - assert len(set(HOOK_EVENTS.values())) == len(HOOK_EVENTS) - - -def test_hook_event_scripts_have_thirdeye_gemini_prefix(): - for script in HOOK_EVENTS.values(): - assert script.startswith("thirdeye-gemini-") - - -def test_hook_timeout_ms(): - assert HOOK_TIMEOUT_MS == 30000 - - -def test_hook_name(): - assert HOOK_NAME == "thirdeye-tracing" diff --git a/tests/test_gemini_hooks.py b/tests/test_gemini_hooks.py deleted file mode 100644 index 62059b3..0000000 --- a/tests/test_gemini_hooks.py +++ /dev/null @@ -1,833 +0,0 @@ -from __future__ import annotations - -import io -import json -from pathlib import Path - -import pytest - -from thirdeye.config import Config -from thirdeye.platforms.gemini import hooks -from thirdeye.store import Store - - -@pytest.fixture -def env(monkeypatch, tmp_path: Path): - monkeypatch.setenv("THIRDEYE_HOME", str(tmp_path)) - return tmp_path - - -def _stdin(monkeypatch, payload: dict) -> None: - monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(payload))) - - -# -- _read_stdin --------------------------------------------------------------- - - -class TestReadStdin: - def test_valid_json(self, monkeypatch): - _stdin(monkeypatch, {"session_id": "abc", "cwd": "/p"}) - result = hooks._read_stdin() - assert result == {"session_id": "abc", "cwd": "/p"} - - def test_empty_stdin_returns_empty_dict(self, monkeypatch): - monkeypatch.setattr("sys.stdin", io.StringIO("")) - assert hooks._read_stdin() == {} - - def test_invalid_json_returns_empty_dict(self, monkeypatch): - monkeypatch.setattr("sys.stdin", io.StringIO("not json")) - assert hooks._read_stdin() == {} - - def test_io_error_returns_empty_dict(self, monkeypatch): - class BrokenStdin: - def read(self): - raise OSError("broken pipe") - - monkeypatch.setattr("sys.stdin", BrokenStdin()) - assert hooks._read_stdin() == {} - - def test_nested_payload(self, monkeypatch): - payload = {"session_id": "s", "nested": {"a": [1, 2, 3]}} - _stdin(monkeypatch, payload) - assert hooks._read_stdin() == payload - - -# -- _flex_get ----------------------------------------------------------------- - - -class TestFlexGet: - def test_returns_first_matching_key(self): - d = {"sessionId": "abc"} - assert hooks._flex_get(d, "session_id", "sessionId") == "abc" - - def test_prefers_first_key_listed(self): - d = {"session_id": "first", "sessionId": "second"} - assert hooks._flex_get(d, "session_id", "sessionId") == "first" - - def test_skips_none_values(self): - d = {"session_id": None, "sessionId": "fallback"} - assert hooks._flex_get(d, "session_id", "sessionId") == "fallback" - - def test_skips_empty_string_values(self): - d = {"session_id": "", "sessionId": "fallback"} - assert hooks._flex_get(d, "session_id", "sessionId") == "fallback" - - def test_returns_default_when_no_match(self): - d = {"other_key": "val"} - assert hooks._flex_get(d, "session_id", "sessionId") is None - - def test_returns_custom_default(self): - d = {} - assert hooks._flex_get(d, "session_id", default="fallback") == "fallback" - - -# -- _strip_payload ------------------------------------------------------------ - - -class TestStripPayload: - def test_removes_routing_keys(self): - result = hooks._strip_payload({"session_id": "abc", "cwd": "/p", "prompt": "hi"}) - assert "session_id" not in result - assert "cwd" not in result - assert result == {"prompt": "hi"} - - def test_removes_camel_case_variants(self): - result = hooks._strip_payload({"sessionId": "abc", "workingDir": "/p", "prompt": "hi"}) - assert "sessionId" not in result - assert "workingDir" not in result - assert result == {"prompt": "hi"} - - def test_removes_working_dir_snake_case(self): - result = hooks._strip_payload({"working_dir": "/p", "data": "val"}) - assert "working_dir" not in result - assert result == {"data": "val"} - - def test_preserves_other_keys(self): - payload = {"session_id": "abc", "tool_name": "Read", "tool_input": {"x": 1}} - result = hooks._strip_payload(payload) - assert result == {"tool_name": "Read", "tool_input": {"x": 1}} - - def test_empty_dict(self): - assert hooks._strip_payload({}) == {} - - def test_only_strip_keys(self): - payload = { - "session_id": "abc", - "sessionId": "abc", - "cwd": "/p", - "workingDir": "/p", - "working_dir": "/p", - } - assert hooks._strip_payload(payload) == {} - - -# -- _emit --------------------------------------------------------------------- - - -class TestEmit: - def test_returns_seq_on_success(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "abc", "cwd": "/p"}) - payload = hooks._read_stdin() - seq = hooks._emit("test_event", payload) - assert isinstance(seq, int) - assert seq >= 0 - - def test_returns_none_without_session_id(self, monkeypatch, env: Path): - assert hooks._emit("test_event", {"cwd": "/p"}) is None - - def test_returns_none_with_empty_session_id(self, monkeypatch, env: Path): - assert hooks._emit("test_event", {"session_id": "", "cwd": "/p"}) is None - - def test_returns_none_with_none_session_id(self, monkeypatch, env: Path): - assert hooks._emit("test_event", {"session_id": None, "cwd": "/p"}) is None - - def test_stores_event_with_correct_type(self, monkeypatch, env: Path): - hooks._emit("my_type", {"session_id": "s1", "cwd": "/p", "key": "val"}) - events = list(Store(Config.load()).reader("s1").iter_events()) - assert len(events) == 1 - assert events[0]["t"] == "my_type" - - def test_strips_routing_keys_from_data(self, monkeypatch, env: Path): - hooks._emit("x", {"session_id": "s1", "cwd": "/p", "extra": 42}) - events = list(Store(Config.load()).reader("s1").iter_events()) - data = events[0].get("data", {}) - assert "session_id" not in data - assert "cwd" not in data - assert data["extra"] == 42 - - def test_uses_cwd_from_payload(self, monkeypatch, env: Path): - hooks._emit("x", {"session_id": "s1", "cwd": "/my/project"}) - m = next(Store(Config.load()).list_sessions()) - assert m.cwd == "/my/project" - - def test_falls_back_to_os_cwd_when_no_cwd(self, monkeypatch, env: Path): - monkeypatch.chdir(env) - hooks._emit("x", {"session_id": "s1"}) - m = next(Store(Config.load()).list_sessions()) - assert m.cwd == str(env) - - def test_accepts_camel_case_session_id(self, monkeypatch, env: Path): - hooks._emit("x", {"sessionId": "camel1", "cwd": "/p"}) - m = next(Store(Config.load()).list_sessions()) - assert m.session_id == "camel1" - - def test_accepts_camel_case_working_dir(self, monkeypatch, env: Path): - hooks._emit("x", {"session_id": "s1", "workingDir": "/camel/dir"}) - m = next(Store(Config.load()).list_sessions()) - assert m.cwd == "/camel/dir" - - def test_platform_is_gemini(self, monkeypatch, env: Path): - hooks._emit("x", {"session_id": "s1", "cwd": "/p"}) - m = next(Store(Config.load()).list_sessions()) - assert m.platform == "gemini" - - -# -- session_start ------------------------------------------------------------- - - -class TestSessionStart: - def test_creates_session(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "abc-123", "cwd": "/proj/x", "source": "cli"}) - hooks.session_start() - store = Store(Config.load()) - metas = list(store.list_sessions()) - assert len(metas) == 1 - m = metas[0] - assert m.session_id == "abc-123" - assert m.platform == "gemini" - assert m.cwd == "/proj/x" - assert m.event_count == 1 - assert m.status == "open" - - def test_event_type_is_session_start(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - hooks.session_start() - events = list(Store(Config.load()).reader("s1").iter_events()) - assert events[0]["t"] == "session_start" - - def test_stores_payload_fields_in_data(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p", "source": "cli"}) - hooks.session_start() - events = list(Store(Config.load()).reader("s1").iter_events()) - assert events[0]["data"]["source"] == "cli" - - -# -- session_start env capture -> auto tags ----------------------------------- - - -class TestSessionStartEnvTags: - def _tags_lines(self, env: Path, sid: str) -> list[dict]: - from thirdeye.paths import session_dir as _sd - from thirdeye.paths import tags_path - - path = tags_path(_sd(env, "gemini", sid)) - if not path.exists(): - return [] - return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] - - def test_no_patterns_set_writes_no_tags(self, monkeypatch, env: Path): - from thirdeye.paths import session_dir as _sd - from thirdeye.paths import tags_path - - monkeypatch.delenv("THIRDEYE_CAPTURE_ENV", raising=False) - monkeypatch.setenv("WB_PLAN", "p") - monkeypatch.setenv("WB_STEP", "test#1") - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - hooks.session_start() - assert not tags_path(_sd(env, "gemini", "s1")).exists() - - def test_matching_env_vars_become_auto_tags(self, monkeypatch, env: Path): - monkeypatch.setenv("THIRDEYE_CAPTURE_ENV", "WB_*") - monkeypatch.setenv("WB_PLAN", "p") - monkeypatch.setenv("WB_STEP", "test#1") - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - hooks.session_start() - - events = list(Store(Config.load()).reader("s1").iter_events()) - start_seq = events[0]["seq"] - - lines = self._tags_lines(env, "s1") - tags = {line["tag"] for line in lines} - assert "plan-p" in tags - assert "step-test#1" in tags - for line in lines: - assert line["op"] == "add" - assert line["source"] == "auto" - assert line["seq"] == start_seq - - def test_invalid_tag_value_is_skipped(self, monkeypatch, env: Path): - monkeypatch.setenv("THIRDEYE_CAPTURE_ENV", "WB_*") - monkeypatch.setenv("WB_PLAN", "p") - monkeypatch.setenv("WB_X", "===") - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - hooks.session_start() - - lines = self._tags_lines(env, "s1") - tags = {line["tag"] for line in lines} - assert "plan-p" in tags - assert not any(t.startswith("x-") for t in tags) - - def test_missing_session_id_writes_no_tags(self, monkeypatch, env: Path): - monkeypatch.setenv("THIRDEYE_CAPTURE_ENV", "WB_*") - monkeypatch.setenv("WB_PLAN", "p") - _stdin(monkeypatch, {"cwd": "/p"}) - hooks.session_start() - assert list(Store(Config.load()).list_sessions()) == [] - - -# -- session_end --------------------------------------------------------------- - - -class TestSessionEnd: - def test_closes_session(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "abc", "cwd": "/p"}) - hooks.session_start() - _stdin(monkeypatch, {"session_id": "abc", "cwd": "/p"}) - hooks.session_end() - store = Store(Config.load()) - m = next(store.list_sessions()) - assert m.status == "closed" - assert m.ended_at is not None - events = list(store.reader("abc").iter_events()) - assert events[-1]["t"] == "session_end" - - def test_appends_event_before_closing(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "abc", "cwd": "/p"}) - hooks.session_start() - _stdin(monkeypatch, {"session_id": "abc", "cwd": "/p"}) - hooks.session_end() - store = Store(Config.load()) - m = next(store.list_sessions()) - assert m.event_count == 2 - - def test_session_end_no_session_id_is_noop(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"cwd": "/p"}) - hooks.session_end() - assert list(Store(Config.load()).list_sessions()) == [] - - -# -- before_agent -------------------------------------------------------------- - - -class TestBeforeAgent: - def test_appends_user_message(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "abc", "cwd": "/p"}) - hooks.session_start() - _stdin(monkeypatch, {"session_id": "abc", "cwd": "/p", "prompt": "hello"}) - hooks.before_agent() - store = Store(Config.load()) - events = list(store.reader("abc").iter_events()) - assert events[0]["t"] == "session_start" - assert events[1]["t"] == "user_message" - assert events[1]["data"]["prompt"] == "hello" - assert "session_id" not in events[1].get("data", {}) - - -# -- before_agent hashtag extraction ------------------------------------------- - - -class TestBeforeAgentHashtagExtract: - def test_extracts_hashtag_from_prompt(self, monkeypatch, env: Path, capsys): - _stdin(monkeypatch, {"sessionId": "g1", "prompt": "let's #ship"}) - hooks.before_agent() - store = Store(Config.load()) - events = list(store.reader("g1").iter_events()) - assert len(events) == 1 - assert events[0]["t"] == "user_message" - - from thirdeye.paths import session_dir as _sd - from thirdeye.paths import tags_path - from thirdeye.tags import TagStore - - sd = _sd(Config.load().root, "gemini", "g1") - tag_file = tags_path(sd) - assert tag_file.exists() - lines = [json.loads(line) for line in tag_file.read_text().splitlines() if line.strip()] - assert len(lines) == 1 - assert lines[0]["tag"] == "ship" - assert lines[0]["op"] == "add" - assert lines[0]["source"] == "auto" - - ts = TagStore(sd) - assert ts.unique_tags() == {"ship"} - - m = next(store.list_sessions()) - assert m.tag_count == 1 - - captured = capsys.readouterr() - assert captured.out == "{}\n" - - def test_no_prompt_no_tags(self, monkeypatch, env: Path, capsys): - _stdin(monkeypatch, {"sessionId": "g2", "cwd": "/p"}) - hooks.before_agent() - - from thirdeye.paths import session_dir as _sd - from thirdeye.paths import tags_path - - sd = _sd(Config.load().root, "gemini", "g2") - assert not tags_path(sd).exists() - - m = next(Store(Config.load()).list_sessions()) - assert m.tag_count == 0 - - captured = capsys.readouterr() - assert captured.out == "{}\n" - - def test_malformed_payload_no_crash(self, monkeypatch, env: Path, capsys): - monkeypatch.setattr("sys.stdin", io.StringIO("not json")) - hooks.before_agent() - assert list(Store(Config.load()).list_sessions()) == [] - captured = capsys.readouterr() - assert captured.out == "{}\n" - - def test_extracts_from_alternate_field_name(self, monkeypatch, env: Path, capsys): - _stdin(monkeypatch, {"sessionId": "g3", "input": "ship it #fast"}) - hooks.before_agent() - - from thirdeye.paths import session_dir as _sd - from thirdeye.tags import TagStore - - sd = _sd(Config.load().root, "gemini", "g3") - ts = TagStore(sd) - assert ts.unique_tags() == {"fast"} - - m = next(Store(Config.load()).list_sessions()) - assert m.tag_count == 1 - - captured = capsys.readouterr() - assert captured.out == "{}\n" - - -# -- after_agent --------------------------------------------------------------- - - -class TestAfterAgent: - def test_appends_assistant_message(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - hooks.session_start() - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p", "response": "done"}) - hooks.after_agent() - events = list(Store(Config.load()).reader("s1").iter_events()) - assert events[1]["t"] == "assistant_message" - assert events[1]["data"]["response"] == "done" - - -# -- before_model -------------------------------------------------------------- - - -class TestBeforeModel: - def test_appends_model_request(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - hooks.session_start() - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p", "model": "gemini-2.5"}) - hooks.before_model() - events = list(Store(Config.load()).reader("s1").iter_events()) - assert events[1]["t"] == "model_request" - assert events[1]["data"]["model"] == "gemini-2.5" - - -# -- after_model --------------------------------------------------------------- - - -class TestAfterModel: - def test_appends_model_response(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - hooks.session_start() - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p", "tokens": 150}) - hooks.after_model() - events = list(Store(Config.load()).reader("s1").iter_events()) - assert events[1]["t"] == "model_response" - assert events[1]["data"]["tokens"] == 150 - - -# -- before_tool --------------------------------------------------------------- - - -class TestBeforeTool: - def test_appends_tool_call(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "abc", "cwd": "/p"}) - hooks.session_start() - _stdin( - monkeypatch, - { - "session_id": "abc", - "cwd": "/p", - "tool_name": "Read", - "tool_input": {"file_path": "x.py"}, - }, - ) - hooks.before_tool() - events = list(Store(Config.load()).reader("abc").iter_events()) - assert events[1]["t"] == "tool_call" - assert events[1]["data"]["tool_name"] == "Read" - assert events[1]["data"]["tool_input"] == {"file_path": "x.py"} - - -# -- after_tool ---------------------------------------------------------------- - - -class TestAfterTool: - def test_appends_tool_result(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "abc", "cwd": "/p"}) - hooks.session_start() - _stdin( - monkeypatch, - { - "session_id": "abc", - "cwd": "/p", - "tool_name": "Read", - "tool_response": "", - }, - ) - hooks.after_tool() - events = list(Store(Config.load()).reader("abc").iter_events()) - assert events[1]["t"] == "tool_result" - assert events[1]["data"]["tool_response"] == "" - - -# -- camelCase sessionId ------------------------------------------------------- - - -class TestCamelCaseSessionId: - def test_session_start_with_camel_case_session_id(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"sessionId": "camel-1", "cwd": "/p"}) - hooks.session_start() - store = Store(Config.load()) - metas = list(store.list_sessions()) - assert len(metas) == 1 - assert metas[0].session_id == "camel-1" - - def test_session_end_with_camel_case_session_id(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"sessionId": "camel-2", "cwd": "/p"}) - hooks.session_start() - _stdin(monkeypatch, {"sessionId": "camel-2", "cwd": "/p"}) - hooks.session_end() - store = Store(Config.load()) - m = next(store.list_sessions()) - assert m.status == "closed" - assert m.ended_at is not None - - def test_before_agent_with_camel_case_session_id(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"sessionId": "camel-3", "cwd": "/p", "prompt": "hi"}) - hooks.before_agent() - events = list(Store(Config.load()).reader("camel-3").iter_events()) - assert events[0]["t"] == "user_message" - - def test_working_dir_camel_case_accepted(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"sessionId": "camel-4", "workingDir": "/camel/path"}) - hooks.session_start() - m = next(Store(Config.load()).list_sessions()) - assert m.cwd == "/camel/path" - - def test_strips_camel_case_keys_from_data(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"sessionId": "camel-5", "workingDir": "/p", "extra": 1}) - hooks.session_start() - events = list(Store(Config.load()).reader("camel-5").iter_events()) - data = events[0].get("data", {}) - assert "sessionId" not in data - assert "workingDir" not in data - assert data["extra"] == 1 - - -# -- stdout contract ----------------------------------------------------------- - - -class TestStdoutContract: - """Gemini hooks MUST print {} to stdout. Absence of stdout breaks Gemini.""" - - def test_session_start_prints_empty_json(self, monkeypatch, env: Path, capsys): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - hooks.session_start() - captured = capsys.readouterr() - assert captured.out == "{}\n" - - def test_session_end_prints_empty_json(self, monkeypatch, env: Path, capsys): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - hooks.session_start() - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - hooks.session_end() - captured = capsys.readouterr() - # session_start prints {}\n, session_end prints {}\n - assert captured.out == "{}\n{}\n" - - def test_before_agent_prints_empty_json(self, monkeypatch, env: Path, capsys): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - hooks.before_agent() - captured = capsys.readouterr() - assert captured.out == "{}\n" - - def test_after_agent_prints_empty_json(self, monkeypatch, env: Path, capsys): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - hooks.after_agent() - captured = capsys.readouterr() - assert captured.out == "{}\n" - - def test_before_model_prints_empty_json(self, monkeypatch, env: Path, capsys): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - hooks.before_model() - captured = capsys.readouterr() - assert captured.out == "{}\n" - - def test_after_model_prints_empty_json(self, monkeypatch, env: Path, capsys): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - hooks.after_model() - captured = capsys.readouterr() - assert captured.out == "{}\n" - - def test_before_tool_prints_empty_json(self, monkeypatch, env: Path, capsys): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - hooks.before_tool() - captured = capsys.readouterr() - assert captured.out == "{}\n" - - def test_after_tool_prints_empty_json(self, monkeypatch, env: Path, capsys): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - hooks.after_tool() - captured = capsys.readouterr() - assert captured.out == "{}\n" - - def test_noop_still_prints_empty_json(self, monkeypatch, env: Path, capsys): - """Even when session_id is missing, stdout must have {}.""" - _stdin(monkeypatch, {"cwd": "/p"}) - hooks.session_start() - captured = capsys.readouterr() - assert captured.out == "{}\n" - - def test_broken_stdin_still_prints_empty_json(self, monkeypatch, env: Path, capsys): - class BrokenStdin: - def read(self): - raise OSError("broken pipe") - - monkeypatch.setattr("sys.stdin", BrokenStdin()) - hooks.before_agent() - captured = capsys.readouterr() - assert captured.out == "{}\n" - - def test_invalid_json_still_prints_empty_json(self, monkeypatch, env: Path, capsys): - monkeypatch.setattr("sys.stdin", io.StringIO("not json")) - hooks.after_model() - captured = capsys.readouterr() - assert captured.out == "{}\n" - - -# -- silent noop edge cases ---------------------------------------------------- - - -class TestSilentNoop: - def test_missing_session_id_is_silent_noop(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"cwd": "/p"}) - hooks.before_agent() - assert list(Store(Config.load()).list_sessions()) == [] - - def test_invalid_json_is_silent_noop(self, monkeypatch, env: Path): - monkeypatch.setattr("sys.stdin", io.StringIO("not json")) - hooks.session_start() - assert list(Store(Config.load()).list_sessions()) == [] - - def test_empty_stdin_is_silent_noop(self, monkeypatch, env: Path): - monkeypatch.setattr("sys.stdin", io.StringIO("")) - hooks.session_start() - assert list(Store(Config.load()).list_sessions()) == [] - - def test_broken_stdin_is_silent_noop(self, monkeypatch, env: Path): - class BrokenStdin: - def read(self): - raise OSError("broken pipe") - - monkeypatch.setattr("sys.stdin", BrokenStdin()) - hooks.before_tool() - assert list(Store(Config.load()).list_sessions()) == [] - - def test_empty_session_id_is_noop(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "", "cwd": "/p"}) - hooks.after_agent() - assert list(Store(Config.load()).list_sessions()) == [] - - def test_null_session_id_is_noop(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": None, "cwd": "/p"}) - hooks.before_model() - assert list(Store(Config.load()).list_sessions()) == [] - - def test_session_end_without_session_does_not_crash(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"cwd": "/p"}) - hooks.session_end() - assert list(Store(Config.load()).list_sessions()) == [] - - def test_noop_still_prints_stdout(self, monkeypatch, env: Path, capsys): - """Even on noop, Gemini requires {} on stdout.""" - _stdin(monkeypatch, {"cwd": "/p"}) - hooks.session_start() - captured = capsys.readouterr() - assert captured.out == "{}\n" - - -# -- all event types route correctly ------------------------------------------- - - -class TestAllEventTypesRoute: - def test_all_event_types_route_correctly(self, monkeypatch, env: Path): - expected = [ - (hooks.session_start, "session_start"), - (hooks.session_end, "session_end"), - (hooks.before_agent, "user_message"), - (hooks.after_agent, "assistant_message"), - (hooks.before_model, "model_request"), - (hooks.after_model, "model_response"), - (hooks.before_tool, "tool_call"), - (hooks.after_tool, "tool_result"), - ] - for fn, t in expected: - _stdin(monkeypatch, {"session_id": "s", "cwd": "/p"}) - fn() - events = list(Store(Config.load()).reader("s").iter_events()) - assert [e["t"] for e in events] == [t for _, t in expected] - - def test_exactly_eight_hooks(self): - hook_fns = [ - hooks.session_start, - hooks.session_end, - hooks.before_agent, - hooks.after_agent, - hooks.before_model, - hooks.after_model, - hooks.before_tool, - hooks.after_tool, - ] - assert len(hook_fns) == 8 - # all are callable - for fn in hook_fns: - assert callable(fn) - - -# -- session_end closes session ------------------------------------------------ - - -class TestSessionEndClosesSession: - def test_status_is_closed(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "abc", "cwd": "/p"}) - hooks.session_start() - _stdin(monkeypatch, {"session_id": "abc", "cwd": "/p"}) - hooks.session_end() - store = Store(Config.load()) - m = next(store.list_sessions()) - assert m.status == "closed" - - def test_ended_at_is_set(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "abc", "cwd": "/p"}) - hooks.session_start() - _stdin(monkeypatch, {"session_id": "abc", "cwd": "/p"}) - hooks.session_end() - store = Store(Config.load()) - m = next(store.list_sessions()) - assert m.ended_at is not None - - def test_does_not_close_without_emit(self, monkeypatch, env: Path): - """session_end should NOT call close_session if _emit returned False.""" - _stdin(monkeypatch, {"cwd": "/p"}) - hooks.session_end() - # no sessions should exist at all - assert list(Store(Config.load()).list_sessions()) == [] - - def test_close_uses_gemini_platform(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "abc", "cwd": "/p"}) - hooks.session_start() - _stdin(monkeypatch, {"session_id": "abc", "cwd": "/p"}) - hooks.session_end() - store = Store(Config.load()) - m = next(store.list_sessions()) - assert m.platform == "gemini" - - def test_close_with_camel_case_session_id(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"sessionId": "camel-close", "cwd": "/p"}) - hooks.session_start() - _stdin(monkeypatch, {"sessionId": "camel-close", "cwd": "/p"}) - hooks.session_end() - store = Store(Config.load()) - m = next(store.list_sessions()) - assert m.status == "closed" - assert m.ended_at is not None - - -# -- platform constant --------------------------------------------------------- - - -class TestPlatformConstant: - def test_platform_is_gemini(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p"}) - hooks.session_start() - m = next(Store(Config.load()).list_sessions()) - assert m.platform == "gemini" - - def test_platform_constant_value(self): - assert hooks._PLATFORM == "gemini" - - -# -- multiple sessions --------------------------------------------------------- - - -class TestMultipleSessions: - def test_different_sessions_are_independent(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/a"}) - hooks.session_start() - _stdin(monkeypatch, {"session_id": "s2", "cwd": "/b"}) - hooks.session_start() - store = Store(Config.load()) - metas = sorted(store.list_sessions(), key=lambda m: m.session_id) - assert len(metas) == 2 - assert metas[0].session_id == "s1" - assert metas[0].cwd == "/a" - assert metas[1].session_id == "s2" - assert metas[1].cwd == "/b" - - def test_closing_one_session_does_not_affect_other(self, monkeypatch, env: Path): - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/a"}) - hooks.session_start() - _stdin(monkeypatch, {"session_id": "s2", "cwd": "/b"}) - hooks.session_start() - _stdin(monkeypatch, {"session_id": "s1", "cwd": "/a"}) - hooks.session_end() - store = Store(Config.load()) - metas = {m.session_id: m for m in store.list_sessions()} - assert metas["s1"].status == "closed" - assert metas["s2"].status == "open" - - -# -- complex payload preservation ---------------------------------------------- - - -class TestPayloadPreservation: - def test_nested_dict_preserved(self, monkeypatch, env: Path): - payload = { - "session_id": "s1", - "cwd": "/p", - "tool_input": {"nested": {"deep": True, "list": [1, 2, 3]}}, - } - _stdin(monkeypatch, payload) - hooks.before_tool() - events = list(Store(Config.load()).reader("s1").iter_events()) - assert events[0]["data"]["tool_input"] == {"nested": {"deep": True, "list": [1, 2, 3]}} - - def test_large_payload(self, monkeypatch, env: Path): - big_data = {"session_id": "s1", "cwd": "/p", "content": "x" * 10000} - _stdin(monkeypatch, big_data) - hooks.after_agent() - events = list(Store(Config.load()).reader("s1").iter_events()) - assert len(events[0]["data"]["content"]) == 10000 - - def test_payload_with_special_chars(self, monkeypatch, env: Path): - payload = { - "session_id": "s1", - "cwd": "/p", - "text": "line1\nline2\ttab\r\nwindows", - } - _stdin(monkeypatch, payload) - hooks.before_agent() - events = list(Store(Config.load()).reader("s1").iter_events()) - assert events[0]["data"]["text"] == "line1\nline2\ttab\r\nwindows" - - def test_unicode_payload(self, monkeypatch, env: Path): - payload = {"session_id": "s1", "cwd": "/p", "text": "hello world"} - _stdin(monkeypatch, payload) - hooks.after_model() - events = list(Store(Config.load()).reader("s1").iter_events()) - assert events[0]["data"]["text"] == "hello world" diff --git a/tests/test_gemini_install.py b/tests/test_gemini_install.py deleted file mode 100644 index bbb06d6..0000000 --- a/tests/test_gemini_install.py +++ /dev/null @@ -1,511 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -from thirdeye.platforms.gemini.constants import ( - HOOK_EVENTS, - HOOK_NAME, - HOOK_TIMEOUT_MS, - SETTINGS_FILE, -) -from thirdeye.platforms.gemini.install import GeminiPlatform - - -class TestGeminiPlatformAttributes: - def test_name_is_gemini(self): - p = GeminiPlatform(settings_file=Path("/fake/settings.json")) - assert p.name == "gemini" - - def test_display_name(self): - p = GeminiPlatform(settings_file=Path("/fake/settings.json")) - assert p.display_name == "Gemini CLI" - - def test_is_platform_subclass(self): - from thirdeye.platforms.base import Platform - - assert issubclass(GeminiPlatform, Platform) - - def test_default_settings_file_matches_constants(self): - p = GeminiPlatform() - assert p._settings_file == SETTINGS_FILE - - -class TestInstallFreshFile: - def test_writes_all_eight_hook_events(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - assert set(settings["hooks"].keys()) == set(HOOK_EVENTS.keys()) - - def test_creates_parent_dir(self, tmp_path: Path): - settings_file = tmp_path / "nested" / "deeper" / "settings.json" - GeminiPlatform(settings_file=settings_file).install() - assert settings_file.exists() - - def test_each_event_has_one_block(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - for event, blocks in settings["hooks"].items(): - assert len(blocks) == 1, f"expected 1 block for {event}" - - def test_block_has_matcher_field(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - for event, blocks in settings["hooks"].items(): - for block in blocks: - assert "matcher" in block, f"missing matcher in block for {event}" - assert block["matcher"] == "" - - def test_hook_has_name_field(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - for blocks in settings["hooks"].values(): - for block in blocks: - for h in block["hooks"]: - assert h["name"] == HOOK_NAME - - def test_hook_has_type_command(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - for blocks in settings["hooks"].values(): - for block in blocks: - for h in block["hooks"]: - assert h["type"] == "command" - - def test_hook_has_command_string(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - for blocks in settings["hooks"].values(): - for block in blocks: - for h in block["hooks"]: - assert "command" in h - assert isinstance(h["command"], str) - - def test_hook_has_timeout(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - for blocks in settings["hooks"].values(): - for block in blocks: - for h in block["hooks"]: - assert h["timeout"] == HOOK_TIMEOUT_MS - - def test_command_contains_script_name(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - for event, script in HOOK_EVENTS.items(): - cmds = [h["command"] for block in settings["hooks"][event] for h in block["hooks"]] - assert any(script in c for c in cmds), f"{script} not in commands for {event}" - - def test_hook_block_structure(self, tmp_path: Path): - """Each block has matcher + hooks list; each hook has type, name, command, timeout.""" - settings_file = tmp_path / "settings.json" - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - for event, blocks in settings["hooks"].items(): - assert isinstance(blocks, list) - for block in blocks: - assert set(block.keys()) == {"matcher", "hooks"} - assert isinstance(block["hooks"], list) - for h in block["hooks"]: - assert set(h.keys()) == {"type", "name", "command", "timeout"} - - def test_output_is_valid_json(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - GeminiPlatform(settings_file=settings_file).install() - json.loads(settings_file.read_text()) - - def test_file_ends_with_newline(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - GeminiPlatform(settings_file=settings_file).install() - assert settings_file.read_text().endswith("\n") - - def test_exactly_eight_events_registered(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - assert len(settings["hooks"]) == 8 - - -class TestInstallIdempotent: - def test_no_duplicate_blocks(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - p = GeminiPlatform(settings_file=settings_file) - p.install() - p.install() - settings = json.loads(settings_file.read_text()) - for event, blocks in settings["hooks"].items(): - assert len(blocks) == 1, f"expected 1 block for {event} after 2 installs" - - def test_no_duplicate_blocks_triple_install(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - p = GeminiPlatform(settings_file=settings_file) - p.install() - p.install() - p.install() - settings = json.loads(settings_file.read_text()) - for event, blocks in settings["hooks"].items(): - assert len(blocks) == 1, f"expected 1 block for {event} after 3 installs" - - def test_content_identical_after_double_install(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - p = GeminiPlatform(settings_file=settings_file) - p.install() - first = settings_file.read_text() - p.install() - second = settings_file.read_text() - assert first == second - - -class TestInstallPreservesExisting: - def test_preserves_unrelated_top_level_keys(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - settings_file.write_text(json.dumps({"security": {"sandboxing": True}, "theme": "dark"})) - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - assert settings["security"] == {"sandboxing": True} - assert settings["theme"] == "dark" - assert "hooks" in settings - - def test_preserves_other_tools_hook_blocks(self, tmp_path: Path): - """Blocks where no inner hook has name == HOOK_NAME should be kept.""" - settings_file = tmp_path / "settings.json" - foreign_block = { - "matcher": "", - "hooks": [ - { - "type": "command", - "name": "other-tracing-tool", - "command": "/usr/bin/other-tool", - "timeout": 60000, - } - ], - } - settings_file.write_text(json.dumps({"hooks": {"SessionStart": [foreign_block]}})) - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - blocks = settings["hooks"]["SessionStart"] - # Should have the foreign block + our block - assert len(blocks) == 2 - names = [h["name"] for block in blocks for h in block["hooks"]] - assert "other-tracing-tool" in names - assert HOOK_NAME in names - - def test_preserves_hooks_for_unknown_events(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - settings_file.write_text( - json.dumps( - { - "hooks": { - "CustomEvent": [ - { - "matcher": "", - "hooks": [ - { - "type": "command", - "name": "custom-hook", - "command": "/custom/hook", - "timeout": 10000, - } - ], - } - ] - } - } - ) - ) - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - assert "CustomEvent" in settings["hooks"] - cmds = [h["command"] for block in settings["hooks"]["CustomEvent"] for h in block["hooks"]] - assert "/custom/hook" in cmds - - -class TestInstallEdgeCases: - def test_handles_empty_file(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - settings_file.write_text("") - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - assert set(settings["hooks"].keys()) == set(HOOK_EVENTS.keys()) - - def test_handles_malformed_json(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - settings_file.write_text("{invalid json") - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - assert set(settings["hooks"].keys()) == set(HOOK_EVENTS.keys()) - - def test_handles_empty_hooks_dict(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - settings_file.write_text(json.dumps({"hooks": {}})) - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - assert set(settings["hooks"].keys()) == set(HOOK_EVENTS.keys()) - - def test_handles_empty_event_list(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - settings_file.write_text(json.dumps({"hooks": {"SessionStart": []}})) - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - assert len(settings["hooks"]["SessionStart"]) == 1 - - def test_existing_block_from_us_is_replaced_not_duplicated(self, tmp_path: Path): - """If a block already has our HOOK_NAME, reinstall should replace it.""" - settings_file = tmp_path / "settings.json" - old_block = { - "matcher": "", - "hooks": [ - { - "type": "command", - "name": HOOK_NAME, - "command": "/old/path/thirdeye-gemini-session-start", - "timeout": 10000, - } - ], - } - settings_file.write_text(json.dumps({"hooks": {"SessionStart": [old_block]}})) - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - blocks = settings["hooks"]["SessionStart"] - our_blocks = [ - block for block in blocks if any(h.get("name") == HOOK_NAME for h in block["hooks"]) - ] - assert len(our_blocks) == 1, "should have exactly one block with our hook name" - # The timeout should be updated to the current constant - assert our_blocks[0]["hooks"][0]["timeout"] == HOOK_TIMEOUT_MS - - def test_nonexistent_file_treated_as_empty(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - assert not settings_file.exists() - GeminiPlatform(settings_file=settings_file).install() - assert settings_file.exists() - settings = json.loads(settings_file.read_text()) - assert set(settings["hooks"].keys()) == set(HOOK_EVENTS.keys()) - - -class TestUninstallRemovesHooks: - def test_removes_all_our_hooks(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - settings_file.write_text(json.dumps({"theme": "dark"})) - p = GeminiPlatform(settings_file=settings_file) - p.install() - p.uninstall() - # After uninstall with only our hooks, hooks key should be gone - settings = json.loads(settings_file.read_text()) - assert "hooks" not in settings - - def test_drops_empty_event_keys(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - settings_file.write_text(json.dumps({"theme": "dark"})) - p = GeminiPlatform(settings_file=settings_file) - p.install() - p.uninstall() - settings = json.loads(settings_file.read_text()) - for event in HOOK_EVENTS: - assert event not in settings.get("hooks", {}) - - def test_drops_empty_hooks_key(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - settings_file.write_text(json.dumps({"theme": "dark"})) - p = GeminiPlatform(settings_file=settings_file) - p.install() - p.uninstall() - settings = json.loads(settings_file.read_text()) - assert "hooks" not in settings - - def test_deletes_file_when_settings_becomes_empty(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - p = GeminiPlatform(settings_file=settings_file) - p.install() - p.uninstall() - assert not settings_file.exists() - - def test_preserves_other_settings_on_uninstall(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - settings_file.write_text(json.dumps({"security": {"sandboxing": True}})) - p = GeminiPlatform(settings_file=settings_file) - p.install() - p.uninstall() - settings = json.loads(settings_file.read_text()) - assert settings["security"] == {"sandboxing": True} - assert "hooks" not in settings - - def test_no_settings_file_is_noop(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - GeminiPlatform(settings_file=settings_file).uninstall() - assert not settings_file.exists() - - def test_uninstall_idempotent(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - p = GeminiPlatform(settings_file=settings_file) - p.install() - p.uninstall() - p.uninstall() # second uninstall should not error - assert not settings_file.exists() - - def test_uninstall_idempotent_with_other_settings(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - settings_file.write_text(json.dumps({"theme": "dark"})) - p = GeminiPlatform(settings_file=settings_file) - p.install() - p.uninstall() - first = settings_file.read_text() - p.uninstall() - second = settings_file.read_text() - assert first == second - - def test_uninstall_then_install_restores(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - p = GeminiPlatform(settings_file=settings_file) - p.install() - first = json.loads(settings_file.read_text()) - p.uninstall() - p.install() - restored = json.loads(settings_file.read_text()) - assert first == restored - - -class TestUninstallPreservesForeign: - def test_leaves_other_tools_blocks_intact(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - foreign_block = { - "matcher": "", - "hooks": [ - { - "type": "command", - "name": "other-tracing-tool", - "command": "/usr/bin/other-tool", - "timeout": 60000, - } - ], - } - settings_file.write_text(json.dumps({"hooks": {"SessionStart": [foreign_block]}})) - p = GeminiPlatform(settings_file=settings_file) - p.install() - p.uninstall() - settings = json.loads(settings_file.read_text()) - assert "SessionStart" in settings["hooks"] - names = [h["name"] for block in settings["hooks"]["SessionStart"] for h in block["hooks"]] - assert "other-tracing-tool" in names - assert HOOK_NAME not in names - - def test_foreign_blocks_on_multiple_events(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - foreign = { - "matcher": "", - "hooks": [ - { - "type": "command", - "name": "foreign-hook", - "command": "/usr/bin/foreign", - "timeout": 10000, - } - ], - } - settings_file.write_text( - json.dumps( - { - "hooks": { - "SessionStart": [foreign], - "BeforeModel": [foreign], - } - } - ) - ) - p = GeminiPlatform(settings_file=settings_file) - p.install() - p.uninstall() - settings = json.loads(settings_file.read_text()) - assert "SessionStart" in settings["hooks"] - assert "BeforeModel" in settings["hooks"] - # Our events that had no foreign blocks should be gone - for event in HOOK_EVENTS: - if event not in ("SessionStart", "BeforeModel"): - assert event not in settings["hooks"] - - def test_uninstall_with_no_thirdeye_hooks_present(self, tmp_path: Path): - settings_file = tmp_path / "settings.json" - settings_file.write_text( - json.dumps( - { - "hooks": { - "SessionStart": [ - { - "matcher": "", - "hooks": [ - { - "type": "command", - "name": "other-tool", - "command": "/other/tool", - "timeout": 5000, - } - ], - } - ] - } - } - ) - ) - GeminiPlatform(settings_file=settings_file).uninstall() - settings = json.loads(settings_file.read_text()) - assert "/other/tool" in [ - h["command"] for block in settings["hooks"]["SessionStart"] for h in block["hooks"] - ] - - -class TestResolveCommandAbsolutePath: - def test_install_uses_absolute_path_when_which_resolves(self, tmp_path: Path, monkeypatch): - settings_file = tmp_path / "settings.json" - - def fake_which(name): - return f"/usr/local/bin/{name}" - - monkeypatch.setattr("thirdeye.platforms.gemini.install.shutil.which", fake_which) - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - for event, script in HOOK_EVENTS.items(): - cmds = [h["command"] for block in settings["hooks"][event] for h in block["hooks"]] - assert cmds == [f"/usr/local/bin/{script}"] - - def test_install_falls_back_to_bare_name_when_which_fails(self, tmp_path: Path, monkeypatch): - settings_file = tmp_path / "settings.json" - monkeypatch.setattr("thirdeye.platforms.gemini.install.shutil.which", lambda _: None) - GeminiPlatform(settings_file=settings_file).install() - settings = json.loads(settings_file.read_text()) - for event, script in HOOK_EVENTS.items(): - cmds = [h["command"] for block in settings["hooks"][event] for h in block["hooks"]] - assert cmds == [script] - - def test_uninstall_removes_absolute_path_hooks(self, tmp_path: Path, monkeypatch): - settings_file = tmp_path / "settings.json" - - def fake_which(name): - return f"/usr/local/bin/{name}" - - monkeypatch.setattr("thirdeye.platforms.gemini.install.shutil.which", fake_which) - p = GeminiPlatform(settings_file=settings_file) - p.install() - p.uninstall() - assert not settings_file.exists() - - def test_idempotent_with_absolute_paths(self, tmp_path: Path, monkeypatch): - settings_file = tmp_path / "settings.json" - - def fake_which(name): - return f"/opt/bin/{name}" - - monkeypatch.setattr("thirdeye.platforms.gemini.install.shutil.which", fake_which) - p = GeminiPlatform(settings_file=settings_file) - p.install() - first = settings_file.read_text() - p.install() - second = settings_file.read_text() - assert first == second diff --git a/tests/test_usage_claude.py b/tests/test_usage_claude.py index b79abb8..fb85567 100644 --- a/tests/test_usage_claude.py +++ b/tests/test_usage_claude.py @@ -230,7 +230,7 @@ def test_capture_advances_offset_with_no_rows(tmp_path: Path) -> None: """Even when no assistant frames are found, the offset must advance past the read bytes.""" transcript = tmp_path / "user_only.jsonl" transcript.write_text( - '{"type":"user","message":{"role":"user","content":"hi"}}\n' '{"type":"meta"}\n' + '{"type":"user","message":{"role":"user","content":"hi"}}\n{"type":"meta"}\n' ) rows = capture_usage_claude( thirdeye_home=tmp_path, diff --git a/tests/test_usage_gemini.py b/tests/test_usage_gemini.py deleted file mode 100644 index 6330b76..0000000 --- a/tests/test_usage_gemini.py +++ /dev/null @@ -1,102 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from thirdeye.paths import session_dir, usage_jsonl_path, usage_log_path -from thirdeye.platforms.gemini.usage import capture_usage_gemini - -FIXTURE = Path(__file__).parent / "fixtures" / "usage" / "gemini_model_response.json" - - -def _payload() -> dict: - return json.loads(FIXTURE.read_text()) - - -def test_capture_from_real_payload(tmp_path: Path) -> None: - rows = capture_usage_gemini( - thirdeye_home=tmp_path, - session_id="127de361", - payload=_payload(), - triggering_seq=5, - ) - assert rows == 1 - line = json.loads( - usage_jsonl_path(session_dir(tmp_path, "gemini", "127de361")).read_text().strip() - ) - assert line["platform"] == "gemini" - assert line["model"] == "gemini-3-flash-preview" - assert line["input_tokens"] == 9582 - assert line["output_tokens"] == 1 - assert line["total_tokens"] == 9748 - assert line["seq"] == 5 - assert line["total_tokens"] != line["input_tokens"] + line["output_tokens"] - - -def test_capture_skips_empty_usage_metadata(tmp_path: Path) -> None: - payload = _payload() - payload["llm_response"]["usageMetadata"] = {} - assert ( - capture_usage_gemini( - thirdeye_home=tmp_path, session_id="abc", payload=payload, triggering_seq=5 - ) - == 0 - ) - assert not usage_jsonl_path(session_dir(tmp_path, "gemini", "abc")).exists() - - -def test_capture_skips_zero_total_tokens(tmp_path: Path) -> None: - payload = _payload() - payload["llm_response"]["usageMetadata"] = { - "promptTokenCount": 0, - "candidatesTokenCount": 0, - "totalTokenCount": 0, - } - assert ( - capture_usage_gemini( - thirdeye_home=tmp_path, session_id="abc", payload=payload, triggering_seq=5 - ) - == 0 - ) - - -def test_capture_uses_unknown_when_model_missing(tmp_path: Path) -> None: - payload = _payload() - payload["llm_request"].pop("model", None) - rows = capture_usage_gemini( - thirdeye_home=tmp_path, session_id="abc", payload=payload, triggering_seq=1 - ) - assert rows == 1 - line = json.loads(usage_jsonl_path(session_dir(tmp_path, "gemini", "abc")).read_text().strip()) - assert line["model"] == "unknown" - - -def test_capture_no_llm_response(tmp_path: Path) -> None: - rows = capture_usage_gemini( - thirdeye_home=tmp_path, - session_id="abc", - payload={"session_id": "abc"}, - triggering_seq=1, - ) - assert rows == 0 - - -def test_safe_capture_swallows_unexpected_error( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - import thirdeye.platforms.gemini.usage as mod - - original = mod.UsageStore - - class Boom(original): # type: ignore[misc] - def append(self, rows): - raise RuntimeError("simulated") - - monkeypatch.setattr(mod, "UsageStore", Boom) - result = capture_usage_gemini( - thirdeye_home=tmp_path, session_id="abc", payload=_payload(), triggering_seq=1 - ) - assert result is None - assert usage_log_path(tmp_path).exists() diff --git a/tests/web/test_evals_session_def_panel.py b/tests/web/test_evals_session_def_panel.py index 668f894..1c0ae94 100644 --- a/tests/web/test_evals_session_def_panel.py +++ b/tests/web/test_evals_session_def_panel.py @@ -92,11 +92,7 @@ def test_panel_filters_to_definition_and_sorts_desc(client, web_store, web_confi ) (web_config.root / "evals" / "defs" / "parity.yaml").write_text( - "name: parity\n" - "description: ''\n" - "default_agent: claude\n" - "directive: |\n" - " parity directive\n" + "name: parity\ndescription: ''\ndefault_agent: claude\ndirective: |\n parity directive\n" ) r = client.get(f"/sessions/{sid}/evals/parity") diff --git a/tests/web/test_routes_evals_read.py b/tests/web/test_routes_evals_read.py index 3378fb7..331f97b 100644 --- a/tests/web/test_routes_evals_read.py +++ b/tests/web/test_routes_evals_read.py @@ -7,11 +7,7 @@ def test_defs_list_shows_saved_defs(client, web_config): (web_config.root / "evals" / "defs" / "foo.yaml").write_text( - "name: foo\n" - "description: test def\n" - "default_agent: claude\n" - "directive: |\n" - " do the thing\n" + "name: foo\ndescription: test def\ndefault_agent: claude\ndirective: |\n do the thing\n" ) r = client.get("/evals/defs") assert r.status_code == 200 @@ -20,11 +16,7 @@ def test_defs_list_shows_saved_defs(client, web_config): def test_def_show_renders_directive(client, web_config): (web_config.root / "evals" / "defs" / "foo.yaml").write_text( - "name: foo\n" - "default_agent: claude\n" - "description: ''\n" - "directive: |\n" - " do the thing\n" + "name: foo\ndefault_agent: claude\ndescription: ''\ndirective: |\n do the thing\n" ) r = client.get("/evals/defs/foo") assert r.status_code == 200 diff --git a/tests/web/test_skeleton_invariants.py b/tests/web/test_skeleton_invariants.py index b58c548..212abe7 100644 --- a/tests/web/test_skeleton_invariants.py +++ b/tests/web/test_skeleton_invariants.py @@ -81,9 +81,9 @@ def test_htmx_vendored_with_version_comment(client): body = r.content # The plan requires the first line to be a vendor comment naming v2.0.4. first_line = body.splitlines()[0] - assert ( - b"htmx" in first_line and b"v2.0.4" in first_line - ), f"missing vendor comment header, got: {first_line!r}" + assert b"htmx" in first_line and b"v2.0.4" in first_line, ( + f"missing vendor comment header, got: {first_line!r}" + ) # File must be more than just a stub: still has to define the htmx global # / canonical attribute string referenced by future templates. assert b"htmx" in body From efd95aaf3c089935254a38d758ed57498fbd7953 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Wed, 5 Aug 2026 23:19:22 -0700 Subject: [PATCH 03/15] Replace fabricated Claude usage fixture with scrubbed real-data subset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prior claude_transcript.jsonl was 4 hand-authored lines with 1 distinct message.id — too small to exercise per-call deduplication, which is why the duplicate-row defect passed a green suite. Replace it with a curated 15-line subset of a real Claude Code 2.1.214 transcript (543 assistant frames in the source), selected to exercise the extractor's edge cases while staying at 17 KB: - one message.id on 6 identical-usage frames (the dedup case) - one frame (also the only requestId==null frame) - inclusive-input arithmetic: sample call input_tokens=2 with cache_read=254643 + cache_creation=504 -> 255149 - 8 distinct message.ids -> 7 expected de-duplicated calls All prose scrubbed (text/thinking/tool input+result, cwd, gitBranch, sessionId, absolute paths). Ships machine-readable claude_transcript.expected.json and a provenance README with pasted jq verification. Co-Authored-By: Claude Opus 4.8 --- tests/fixtures/usage/README.md | 104 ++++++++++++++++++ .../usage/claude_transcript.expected.json | 20 ++++ tests/fixtures/usage/claude_transcript.jsonl | 19 +++- 3 files changed, 139 insertions(+), 4 deletions(-) create mode 100644 tests/fixtures/usage/README.md create mode 100644 tests/fixtures/usage/claude_transcript.expected.json diff --git a/tests/fixtures/usage/README.md b/tests/fixtures/usage/README.md new file mode 100644 index 0000000..f0517d0 --- /dev/null +++ b/tests/fixtures/usage/README.md @@ -0,0 +1,104 @@ +# Claude Code usage fixtures + +## `claude_transcript.jsonl` + +A **scrubbed, curated subset** of a real Claude Code 2.1.214 session transcript, +used to test the Claude usage extractor's per-call deduplication and inclusive +token arithmetic. It is not the whole transcript: the full session held hundreds +of frames and over a megabyte of prose. Shipping that crashed downstream tooling, +so only the frames needed to exercise the extractor's edge cases are retained, +with original line order preserved. + +### Provenance + +- **Source:** a Claude Code project transcript, + `~/.claude/projects/scrubbed/16a1e984-1dbd-454b-b9f2-0a9e4b4683d3.jsonl` + (project directory name scrubbed for privacy; this repository is public). +- **CLI version:** `2.1.214 (Claude Code)` (the `version` field carried on every frame). + +### Scrubbing rules applied + +Preserved exactly (assertions depend on them): `type`, `timestamp`, `uuid`, +`parentUuid`, `requestId`, `message.id`, `message.model`, `message.role`, +`message.usage` (every subfield), and the `type` of each `message.content[]` +item. + +Replaced: + +- every `message.content[].text` → `"[scrubbed]"` +- every `tool_use` item's `.input` → `{}` +- every `tool_result` item's `.content` → `"[scrubbed]"` +- top-level `toolUseResult` payloads → `"[scrubbed]"` +- `cwd` → `"/scrubbed"`, `gitBranch` → `"main"` +- `sessionId` → a fixed fake UUID (`00000000-0000-4000-8000-000000000000`) +- any absolute filesystem path anywhere → `/scrubbed`, and any residual + personal-name token → `scrubbed` + +Dropped entirely (the extractor ignores them and they carry the most content): +`attachment`, `file-history-snapshot`, `last-prompt`, and `queue-operation` +frames. + +### Verified counts (measured on the shipped fixture) + +| Property | Value | +|---|---| +| total lines | 15 | +| `type=="assistant"` frames | 13 | +| distinct `message.id` among assistant frames | 8 | +| assistant frames with `message.model == ""` | 1 | +| assistant frames with `requestId == null` | 1 | +| most-repeated `message.id` | `msg_011CdBpZs1PvZ3gsGPM8rXdf` | +| ...its frame count | 6 | +| expected de-duplicated calls (`distinct − synthetic`) | 7 | + +### Sample call — `msg_011CdBpZs1PvZ3gsGPM8rXdf` + +| Field | Value | +|---|---| +| `input_tokens` | 2 | +| `output_tokens` | 3195 | +| `cache_read_input_tokens` | 254643 | +| `cache_creation_input_tokens` | 504 | +| **computed inclusive input** (`input + cache_read + cache_creation`) | 255149 | + +The inclusive-input total is what `gen_ai.usage.input_tokens` must equal for this +call: Anthropic reports `input_tokens` *excluding* cache, so the extractor adds +`cache_read_input_tokens` and `cache_creation_input_tokens` back in. + +### Verification (real `jq` output) + +``` +$ wc -l -c tests/fixtures/usage/claude_transcript.jsonl + 15 17291 tests/fixtures/usage/claude_transcript.jsonl + +$ jq -rc 'select(.type=="assistant")' … | wc -l # assistant frames +13 +$ jq -rc 'select(.type=="assistant").message.id' … | sort -u | wc -l # distinct ids +8 +$ jq -rc 'select(.type=="assistant" and .message.model=="")' … | wc -l +1 +$ jq -rc 'select(.type=="assistant" and .requestId==null)' … | wc -l +1 +$ jq -rc 'select(.type=="assistant").message.id' … | sort | uniq -c | sort -rn + 6 msg_011CdBpZs1PvZ3gsGPM8rXdf + 1 msg_011CdATG2uFx1p3cezx4pJh2 + 1 msg_011CdATFYGKcS9wwCERfd2xZ + 1 msg_011CdATFPLJn9tySKE1hqKHk + 1 msg_011CdATFhedZkNVu3DiBHFaa + 1 msg_011CdATFBHktRHGavvpHssof + 1 msg_011CdATEmRK5u9k4QLvKAA2f + 1 1b3a9ee4-60ff-49bc-b876-99790a06f70f + +$ jq -c 'select(.message.id=="msg_011CdBpZs1PvZ3gsGPM8rXdf").message.usage + | {input_tokens,output_tokens,cache_read_input_tokens,cache_creation_input_tokens}' … | head -1 +{"input_tokens":2,"output_tokens":3195,"cache_read_input_tokens":254643,"cache_creation_input_tokens":504} +``` + +Exactly one `message.id` (`msg_011CdBpZs1PvZ3gsGPM8rXdf`) appears on ≥4 frames — +six identical-usage frames, the per-call deduplication case. All six carry the +same `message.usage`, so a naive extractor would emit six rows for one API call. + +## `claude_transcript.expected.json` + +The machine-readable companion the extractor tests load. Every value is measured +from the shipped `claude_transcript.jsonl`, not copied from a specification. diff --git a/tests/fixtures/usage/claude_transcript.expected.json b/tests/fixtures/usage/claude_transcript.expected.json new file mode 100644 index 0000000..644a0cb --- /dev/null +++ b/tests/fixtures/usage/claude_transcript.expected.json @@ -0,0 +1,20 @@ +{ + "source": "~/.claude/projects/scrubbed/16a1e984-1dbd-454b-b9f2-0a9e4b4683d3.jsonl", + "cli_version": "2.1.214 (Claude Code)", + "total_lines": 15, + "assistant_frames": 13, + "distinct_message_ids": 8, + "synthetic_frames": 1, + "null_request_id_frames": 1, + "expected_calls": 7, + "repeated_message_id": "msg_011CdBpZs1PvZ3gsGPM8rXdf", + "repeated_message_id_frame_count": 6, + "sample_call": { + "message_id": "msg_011CdBpZs1PvZ3gsGPM8rXdf", + "input_tokens": 2, + "cache_read_input_tokens": 254643, + "cache_creation_input_tokens": 504, + "output_tokens": 3195, + "expected_inclusive_input_tokens": 255149 + } +} diff --git a/tests/fixtures/usage/claude_transcript.jsonl b/tests/fixtures/usage/claude_transcript.jsonl index 3764261..f278c37 100644 --- a/tests/fixtures/usage/claude_transcript.jsonl +++ b/tests/fixtures/usage/claude_transcript.jsonl @@ -1,4 +1,15 @@ -{"type":"user","message":{"role":"user","content":"hi"},"timestamp":"2026-05-15T00:00:00Z"} -{"type":"assistant","message":{"role":"assistant","model":"claude-opus-4-7","content":[{"type":"text","text":"hello"}],"usage":{"input_tokens":12450,"output_tokens":187},"timestamp":"2026-05-15T00:00:01Z"}} -{"type":"meta","message":null} -{"type":"assistant","message":{"role":"assistant","model":"claude-opus-4-7","content":[{"type":"text","text":"more"}],"usage":{"input_tokens":13082,"output_tokens":91},"timestamp":"2026-05-15T00:00:02Z"}} +{"parentUuid":"592f713d-36c8-4270-8d32-c7704004617f","isSidechain":false,"promptId":"1c3051e1-cb7e-410f-9849-b07795fe541f","type":"user","message":{"role":"user","content":[{"type":"text","text":"[scrubbed]"}]},"uuid":"d74a7dbd-f358-4be2-bd5d-ea8537f34d8b","timestamp":"2026-07-19T00:15:54.511Z","permissionMode":"auto","origin":{"kind":"human"},"promptSource":"sdk","userType":"external","entrypoint":"claude-vscode","cwd":"/scrubbed","sessionId":"00000000-0000-4000-8000-000000000000","version":"2.1.214","gitBranch":"main"} +{"parentUuid":"a7a399dc-3636-48a0-a11a-18556a76e683","isSidechain":false,"message":{"model":"claude-sonnet-5","id":"msg_011CdATEmRK5u9k4QLvKAA2f","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"[scrubbed]","signature":"[scrubbed]"}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":20317,"cache_read_input_tokens":21106,"output_tokens":465,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":20317,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":465,"cache_read_input_tokens":21106,"cache_creation_input_tokens":20317,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":20317},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CdATEkS36LtF79wziQoCL","type":"assistant","uuid":"5eeda6c9-76fc-4d30-ad81-fc1074491d62","timestamp":"2026-07-19T00:15:58.331Z","effort":"high","userType":"external","entrypoint":"claude-vscode","cwd":"/scrubbed","sessionId":"00000000-0000-4000-8000-000000000000","version":"2.1.214","gitBranch":"main"} +{"parentUuid":"4cc26c20-44b1-4726-beab-3c5f507544f3","isSidechain":false,"promptId":"1c3051e1-cb7e-410f-9849-b07795fe541f","type":"user","message":{"role":"user","content":[{"type":"tool_result","content":"[scrubbed]","is_error":true,"tool_use_id":"toolu_0159Hzn9RtAzkRwE22yEmJhz"}]},"uuid":"1340bbcf-7e5e-4a7c-9d34-c80612ea02d9","timestamp":"2026-07-19T00:15:59.786Z","toolUseResult":"[scrubbed]","sourceToolAssistantUUID":"4cc26c20-44b1-4726-beab-3c5f507544f3","userType":"external","entrypoint":"claude-vscode","cwd":"/scrubbed","sessionId":"00000000-0000-4000-8000-000000000000","version":"2.1.214","gitBranch":"main"} +{"parentUuid":"fc998726-941a-4bc2-8c9c-c75fdb95d070","isSidechain":false,"message":{"model":"claude-sonnet-5","id":"msg_011CdATFBHktRHGavvpHssof","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"[scrubbed]","signature":"[scrubbed]"}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":1043,"cache_read_input_tokens":41423,"output_tokens":202,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":1043,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":202,"cache_read_input_tokens":41423,"cache_creation_input_tokens":1043,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":1043},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CdATFAHV8Sf2HGKDCesBh","type":"assistant","uuid":"7d5cc529-1c7c-4cfb-bc93-34901367ab20","timestamp":"2026-07-19T00:16:01.710Z","effort":"high","userType":"external","entrypoint":"claude-vscode","cwd":"/scrubbed","sessionId":"00000000-0000-4000-8000-000000000000","version":"2.1.214","gitBranch":"main"} +{"parentUuid":"b01e48a4-441a-45ca-92a6-a1bd8f0ad972","isSidechain":false,"message":{"model":"claude-sonnet-5","id":"msg_011CdATFPLJn9tySKE1hqKHk","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"[scrubbed]","signature":"[scrubbed]"}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":606,"cache_read_input_tokens":42466,"output_tokens":86,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":606,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":86,"cache_read_input_tokens":42466,"cache_creation_input_tokens":606,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":606},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CdATFMuELHdUxNPYUARAL","type":"assistant","uuid":"20177bfa-a314-4913-9871-426ff818b2e8","timestamp":"2026-07-19T00:16:04.988Z","effort":"high","userType":"external","entrypoint":"claude-vscode","cwd":"/scrubbed","sessionId":"00000000-0000-4000-8000-000000000000","version":"2.1.214","gitBranch":"main"} +{"parentUuid":"65a3302d-b902-4426-b341-fd542c21e286","isSidechain":false,"message":{"model":"claude-sonnet-5","id":"msg_011CdATFYGKcS9wwCERfd2xZ","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_0178MZJHcHMVKWFmbVvR9ZFy","name":"Read","input":{},"caller":{"type":"direct"}}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":127,"cache_read_input_tokens":43072,"output_tokens":115,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":127,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":115,"cache_read_input_tokens":43072,"cache_creation_input_tokens":127,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":127},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CdATFXDZpnH5opG3TL1dV","type":"assistant","uuid":"e479ad93-6190-45c8-979a-44dc0c7bb72a","timestamp":"2026-07-19T00:16:06.801Z","effort":"high","userType":"external","entrypoint":"claude-vscode","cwd":"/scrubbed","sessionId":"00000000-0000-4000-8000-000000000000","version":"2.1.214","gitBranch":"main"} +{"parentUuid":"607457d1-7bce-42a8-b90c-0816e0213e46","isSidechain":false,"message":{"model":"claude-sonnet-5","id":"msg_011CdATFhedZkNVu3DiBHFaa","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"[scrubbed]","signature":"[scrubbed]"}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":2835,"cache_read_input_tokens":43199,"output_tokens":277,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":2835,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":277,"cache_read_input_tokens":43199,"cache_creation_input_tokens":2835,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":2835},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CdATFf859eHNgbBuUVtgj","type":"assistant","uuid":"81c321ee-85a0-4192-a2c4-59210f585495","timestamp":"2026-07-19T00:16:11.044Z","effort":"high","userType":"external","entrypoint":"claude-vscode","cwd":"/scrubbed","sessionId":"00000000-0000-4000-8000-000000000000","version":"2.1.214","gitBranch":"main"} +{"parentUuid":"cc2744ed-dc9b-4b53-b8fc-07b02d9d91e9","isSidechain":false,"message":{"model":"claude-sonnet-5","id":"msg_011CdATG2uFx1p3cezx4pJh2","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"[scrubbed]","signature":"[scrubbed]"}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":341,"cache_read_input_tokens":46034,"output_tokens":145,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":341,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":145,"cache_read_input_tokens":46034,"cache_creation_input_tokens":341,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":341},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CdATG21BrwUCYVBXH3W5W","type":"assistant","uuid":"ad575e78-0149-490e-a8fb-307e5e6b2222","timestamp":"2026-07-19T00:16:13.999Z","effort":"high","userType":"external","entrypoint":"claude-vscode","cwd":"/scrubbed","sessionId":"00000000-0000-4000-8000-000000000000","version":"2.1.214","gitBranch":"main"} +{"parentUuid":"4738b6d7-9819-4a7c-a028-de3162db7803","isSidechain":false,"message":{"model":"claude-fable-5","id":"msg_011CdBpZs1PvZ3gsGPM8rXdf","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"[scrubbed]","signature":"[scrubbed]"}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":504,"cache_read_input_tokens":254643,"output_tokens":3195,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":504,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":3195,"cache_read_input_tokens":254643,"cache_creation_input_tokens":504,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":504},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CdBpZoQMSSfTP6yn8txG9","type":"assistant","uuid":"451c5161-f103-4777-b347-db1ff8588f2c","timestamp":"2026-07-19T17:36:14.708Z","effort":"high","userType":"external","entrypoint":"claude-vscode","cwd":"/scrubbed","sessionId":"00000000-0000-4000-8000-000000000000","version":"2.1.214","gitBranch":"main"} +{"parentUuid":"451c5161-f103-4777-b347-db1ff8588f2c","isSidechain":false,"message":{"model":"claude-fable-5","id":"msg_011CdBpZs1PvZ3gsGPM8rXdf","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01BPzMk8GcrvNsk5p9fK3ghY","name":"Write","input":{},"caller":{"type":"direct"}}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":504,"cache_read_input_tokens":254643,"output_tokens":3195,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":504,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":3195,"cache_read_input_tokens":254643,"cache_creation_input_tokens":504,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":504},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CdBpZoQMSSfTP6yn8txG9","type":"assistant","uuid":"33beb827-2ad7-45e2-a634-a7a1f8ecc13c","timestamp":"2026-07-19T17:36:21.767Z","effort":"high","userType":"external","entrypoint":"claude-vscode","cwd":"/scrubbed","sessionId":"00000000-0000-4000-8000-000000000000","version":"2.1.214","gitBranch":"main"} +{"parentUuid":"35f93f64-bdad-4502-ba56-3384bacab3b4","isSidechain":false,"message":{"model":"claude-fable-5","id":"msg_011CdBpZs1PvZ3gsGPM8rXdf","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01X7zSSnoqmua6RXCTCwr2Cp","name":"Write","input":{},"caller":{"type":"direct"}}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":504,"cache_read_input_tokens":254643,"output_tokens":3195,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":504,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":3195,"cache_read_input_tokens":254643,"cache_creation_input_tokens":504,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":504},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CdBpZoQMSSfTP6yn8txG9","type":"assistant","uuid":"0cadea09-ee36-49b4-abaa-f7f63d853fe9","timestamp":"2026-07-19T17:36:26.945Z","effort":"high","userType":"external","entrypoint":"claude-vscode","cwd":"/scrubbed","sessionId":"00000000-0000-4000-8000-000000000000","version":"2.1.214","gitBranch":"main"} +{"parentUuid":"5e6086fd-cd70-4f41-a716-c06e0f8fb30a","isSidechain":false,"message":{"model":"claude-fable-5","id":"msg_011CdBpZs1PvZ3gsGPM8rXdf","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01TNw7qVpSiNhPUGMKyAhLqQ","name":"Write","input":{},"caller":{"type":"direct"}}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":504,"cache_read_input_tokens":254643,"output_tokens":3195,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":504,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":3195,"cache_read_input_tokens":254643,"cache_creation_input_tokens":504,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":504},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CdBpZoQMSSfTP6yn8txG9","type":"assistant","uuid":"3fb1d11d-fb23-4eec-98e0-ebd7c426648b","timestamp":"2026-07-19T17:36:34.566Z","effort":"high","userType":"external","entrypoint":"claude-vscode","cwd":"/scrubbed","sessionId":"00000000-0000-4000-8000-000000000000","version":"2.1.214","gitBranch":"main"} +{"parentUuid":"cf9a9786-d347-4683-8b22-b9f6120b4784","isSidechain":false,"message":{"model":"claude-fable-5","id":"msg_011CdBpZs1PvZ3gsGPM8rXdf","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01V1yivbcS3xEHis39eTZiip","name":"Write","input":{},"caller":{"type":"direct"}}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":504,"cache_read_input_tokens":254643,"output_tokens":3195,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":504,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":3195,"cache_read_input_tokens":254643,"cache_creation_input_tokens":504,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":504},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CdBpZoQMSSfTP6yn8txG9","type":"assistant","uuid":"1c05b464-930a-462e-94db-04e528e081dd","timestamp":"2026-07-19T17:36:36.410Z","effort":"high","userType":"external","entrypoint":"claude-vscode","cwd":"/scrubbed","sessionId":"00000000-0000-4000-8000-000000000000","version":"2.1.214","gitBranch":"main"} +{"parentUuid":"4988b1cf-75aa-427d-a82b-38e11da8a8c1","isSidechain":false,"message":{"model":"claude-fable-5","id":"msg_011CdBpZs1PvZ3gsGPM8rXdf","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01KbsDKSpeSDTz3K5RaV8g91","name":"Write","input":{},"caller":{"type":"direct"}}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":504,"cache_read_input_tokens":254643,"output_tokens":3195,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":504,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":3195,"cache_read_input_tokens":254643,"cache_creation_input_tokens":504,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":504},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CdBpZoQMSSfTP6yn8txG9","type":"assistant","uuid":"ca462bd3-4034-42c3-a9eb-507237cd1753","timestamp":"2026-07-19T17:36:39.066Z","effort":"high","userType":"external","entrypoint":"claude-vscode","cwd":"/scrubbed","sessionId":"00000000-0000-4000-8000-000000000000","version":"2.1.214","gitBranch":"main"} +{"parentUuid":"9947e2d6-2cdb-4037-99f0-389aa90a9c21","isSidechain":false,"type":"assistant","uuid":"2159f3fb-40ca-44f2-82a3-aa5bc2edaa01","timestamp":"2026-07-19T17:47:21.208Z","message":{"id":"1b3a9ee4-60ff-49bc-b876-99790a06f70f","container":null,"model":"","role":"assistant","stop_details":null,"stop_reason":"stop_sequence","stop_sequence":"","type":"message","usage":{"input_tokens":0,"output_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":null,"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":null,"iterations":null,"speed":null},"content":[{"type":"text","text":"[scrubbed]"}],"context_management":null},"error":"invalid_request","errorDetails":"Image base64 size (10.2MB) exceeds API limit (5MB). Please resize the image before sending.","isApiErrorMessage":true,"healsDistinctCarrier":true,"userType":"external","entrypoint":"claude-vscode","cwd":"/scrubbed","sessionId":"00000000-0000-4000-8000-000000000000","version":"2.1.214","gitBranch":"main"} From e7486f7d8ec25c7e653d59ab82c507e453ae9f8b Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Wed, 5 Aug 2026 23:26:53 -0700 Subject: [PATCH 04/15] Add canonical usage read path with call-level dedup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce usage/read.py: iter_calls collapses raw sidecar rows into one row per distinct (session_id, call_id), last-wins, in first-appearance order; call_totals sums input/output tokens over rows. Dedup lives here alone — UsageStore.iter_rows stays a faithful raw mirror, documented as such and guarded by a test asserting duplicates survive. Update test_usage_store.py make_row to the OTel GenAI UsageRow shape. Co-Authored-By: Claude Opus 4.8 --- src/thirdeye/usage/read.py | 49 ++++++++++++++++ src/thirdeye/usage/store.py | 7 ++- tests/test_usage_read.py | 114 ++++++++++++++++++++++++++++++++++++ tests/test_usage_store.py | 18 +++++- 4 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 src/thirdeye/usage/read.py create mode 100644 tests/test_usage_read.py diff --git a/src/thirdeye/usage/read.py b/src/thirdeye/usage/read.py new file mode 100644 index 0000000..348afc7 --- /dev/null +++ b/src/thirdeye/usage/read.py @@ -0,0 +1,49 @@ +"""The single canonical read path over usage sidecars. + +The sidecar (``usage.jsonl``) is a faithful raw mirror: writers append one row +per source frame and never read-modify-write, so the same logical LLM call can +appear many times. Claude repeats the identical ``message.usage`` across every +content-block frame of one API call; Codex emits byte-identical repeat reports. +``iter_calls`` collapses those duplicates into one row per distinct +``(session_id, call_id)``. + +This is why capture needs no locking anywhere: two triggers racing on the same +transcript offset append the same rows, and their identical ``call_id``s +collapse here on read. Deduplication happens only in this module — every +consumer that wants logical calls (as opposed to the raw mirror) goes through +``iter_calls``. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Iterator +from pathlib import Path + +from thirdeye.usage.store import UsageStore +from thirdeye.usage.types import UsageRow + + +def iter_calls(session_dir_: Path) -> Iterator[UsageRow]: + """Yield one row per distinct (session_id, call_id); last occurrence wins. + + Reads the raw sidecar via UsageStore.iter_rows() and collapses duplicates. + Row order follows first appearance of each call_id, so output is stable and + roughly chronological even though the surviving value is the last one seen. + + Last-wins is safe because duplicate rows for one call_id carry identical + token values, so which copy survives cannot change any result. + """ + latest: dict[tuple[str, str], UsageRow] = {} + for row in UsageStore(session_dir_).iter_rows(): + latest[(row.session_id, row.call_id)] = row + yield from latest.values() + + +def call_totals(rows: Iterable[UsageRow]) -> tuple[int, int]: + """Return (input_tokens, output_tokens) summed over rows.""" + input_total = 0 + output_total = 0 + for row in rows: + input_total += row.input_tokens + output_total += row.output_tokens + return input_total, output_total diff --git a/src/thirdeye/usage/store.py b/src/thirdeye/usage/store.py index 1eb5531..465fe36 100644 --- a/src/thirdeye/usage/store.py +++ b/src/thirdeye/usage/store.py @@ -39,7 +39,12 @@ def append(self, rows: list[UsageRow]) -> None: f.write(json.dumps(row.to_dict(), separators=(",", ":")) + "\n") def iter_rows(self) -> Iterator[UsageRow]: - """Yield rows from usage.jsonl. Skips empty / malformed lines silently.""" + """Yield rows from usage.jsonl. Skips empty / malformed lines silently. + + Raw and undeduplicated: this is the faithful mirror of the sidecar, so + the same logical call may appear many times. Callers wanting one row per + logical LLM call must use ``read.iter_calls``. + """ if not self.jsonl_path.exists(): return with self.jsonl_path.open("r", encoding="utf-8") as f: diff --git a/tests/test_usage_read.py b/tests/test_usage_read.py new file mode 100644 index 0000000..6dc936d --- /dev/null +++ b/tests/test_usage_read.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from thirdeye.usage.read import call_totals, iter_calls +from thirdeye.usage.store import UsageStore +from thirdeye.usage.types import UsageRow + + +@pytest.fixture +def session(tmp_path: Path) -> Path: + sd = tmp_path / "traces" / "claude" / "abc123" + sd.mkdir(parents=True) + return sd + + +def make_row(call_id: str, seq: int = 0, **overrides) -> UsageRow: + defaults = dict( + session_id="abc123", + seq=seq, + call_id=call_id, + ts=f"2026-05-15T00:00:{seq:02d}.000Z", + platform="claude", + provider_name="anthropic", + response_model="claude-opus-4-7", + input_tokens=100, + output_tokens=10, + ) + defaults.update(overrides) + return UsageRow(**defaults) + + +def test_collapses_duplicate_call_id(session: Path) -> None: + store = UsageStore(session) + store.append([make_row("call-a") for _ in range(6)]) + rows = list(iter_calls(session)) + assert len(rows) == 1 + assert rows[0].call_id == "call-a" + + +def test_distinct_calls_in_first_appearance_order(session: Path) -> None: + store = UsageStore(session) + store.append([make_row("a", 0), make_row("b", 1), make_row("c", 2)]) + rows = list(iter_calls(session)) + assert [r.call_id for r in rows] == ["a", "b", "c"] + + +def test_first_appearance_order_survives_interleaved_duplicates(session: Path) -> None: + store = UsageStore(session) + store.append( + [ + make_row("a", 0), + make_row("b", 1), + make_row("a", 2), # duplicate of the first call, seen later + make_row("c", 3), + ] + ) + rows = list(iter_calls(session)) + assert [r.call_id for r in rows] == ["a", "b", "c"] + + +def test_last_wins(session: Path) -> None: + store = UsageStore(session) + store.append([make_row("a", 0, output_tokens=10), make_row("a", 1, output_tokens=99)]) + rows = list(iter_calls(session)) + assert len(rows) == 1 + assert rows[0].output_tokens == 99 + + +def test_missing_sidecar_yields_nothing(session: Path) -> None: + assert list(iter_calls(session)) == [] + + +def test_empty_sidecar_yields_nothing(session: Path) -> None: + (session / "usage.jsonl").write_text("") + assert list(iter_calls(session)) == [] + + +def test_malformed_line_skipped_valid_rows_survive(session: Path) -> None: + (session / "usage.jsonl").write_text( + json.dumps(make_row("a", 0).to_dict()) + + "\n" + + "not valid json\n" + + json.dumps(make_row("b", 1).to_dict()) + + "\n" + ) + rows = list(iter_calls(session)) + assert [r.call_id for r in rows] == ["a", "b"] + + +def test_call_totals_sums_deduplicated_rows(session: Path) -> None: + store = UsageStore(session) + store.append([make_row("a", input_tokens=100, output_tokens=10) for _ in range(6)]) + assert call_totals(iter_calls(session)) == (100, 10) + + +def test_call_totals_sums_distinct_rows(session: Path) -> None: + rows = [ + make_row("a", input_tokens=100, output_tokens=10), + make_row("b", input_tokens=50, output_tokens=5), + ] + assert call_totals(rows) == (150, 15) + + +def test_optional_attributes_absent_round_trip_as_none(session: Path) -> None: + store = UsageStore(session) + store.append([make_row("a")]) + (row,) = list(iter_calls(session)) + assert row.cache_read_input_tokens is None + assert row.cache_creation_input_tokens is None + assert row.reasoning_output_tokens is None diff --git a/tests/test_usage_store.py b/tests/test_usage_store.py index f733fde..4d8c3da 100644 --- a/tests/test_usage_store.py +++ b/tests/test_usage_store.py @@ -20,12 +20,13 @@ def make_row(seq: int, **overrides) -> UsageRow: defaults = dict( session_id="abc123", seq=seq, + call_id=f"call-{seq}", ts=f"2026-05-15T00:00:{seq:02d}.000Z", platform="claude", - model="claude-opus-4-7", + provider_name="anthropic", + response_model="claude-opus-4-7", input_tokens=100, output_tokens=10, - total_tokens=110, ) defaults.update(overrides) return UsageRow(**defaults) @@ -77,6 +78,19 @@ def test_iter_rows_skips_malformed_lines(session: Path) -> None: assert [r.seq for r in rows] == [0, 1] +def test_iter_rows_returns_duplicates_raw(session: Path) -> None: + """The sidecar is a faithful raw mirror: iter_rows must NOT deduplicate. + + If dedup ever leaks into the writer or store, this fails — the collapse of + duplicate call_ids belongs only in read.iter_calls. + """ + store = UsageStore(session) + store.append([make_row(0, call_id="dup") for _ in range(6)]) + rows = list(store.iter_rows()) + assert len(rows) == 6 + assert all(r.call_id == "dup" for r in rows) + + def test_read_state_missing_returns_empty(session: Path) -> None: assert UsageStore(session).read_state() == {} From 0c59918e99d86fd622e52400520f4a3093da4cc1 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Wed, 5 Aug 2026 23:28:22 -0700 Subject: [PATCH 05/15] Wire six missing Claude hook events (10 -> 16) Claude Code 2.1.195 exports 16 hook events; thirdeye only listened for 10. Add PostToolUseFailure, SubagentStart, UserPromptExpansion, PreCompact, PostCompact, and PermissionDenied. - constants.py: six new HOOK_EVENTS entries with thirdeye-claude-* names - hooks.py: six one-line _emit handlers. PostToolUseFailure emits tool_result (not error) so failed calls pair with their tool_call in the web view; SubagentStart emits subagent_start while SubagentStop keeps subagent_message to avoid orphaning recorded sessions. Both asymmetries are commented as intentional. - pyproject.toml: six matching [project.scripts] entry points - tests: constants (16 entries, script names, tomllib entry-point cross-check) and hooks (mapped type, key stripping, session_id no-op, trigger preservation) for the new events test_claude_install.py::test_all_ten_events_registered hardcoded a count of 10, which is unavoidably stale once HOOK_EVENTS grows to 16 (install() enumerates HOOK_EVENTS). The install path itself is unchanged; updated the single assertion to derive the count from HOOK_EVENTS and renamed it test_all_events_registered. Co-Authored-By: Claude Opus 4.8 --- pyproject.toml | 6 +++ src/thirdeye/platforms/claude/constants.py | 6 +++ src/thirdeye/platforms/claude/hooks.py | 31 +++++++++++ tests/test_claude_constants.py | 34 ++++++++++++ tests/test_claude_hooks.py | 61 ++++++++++++++++++++++ tests/test_claude_install.py | 4 +- 6 files changed, 140 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2dedfb0..c488e76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,12 @@ thirdeye-claude-stop-failure = "thirdeye.platforms.claude.hooks:stop_failure" thirdeye-claude-notification = "thirdeye.platforms.claude.hooks:notification" thirdeye-claude-permission-request = "thirdeye.platforms.claude.hooks:permission_request" thirdeye-claude-session-end = "thirdeye.platforms.claude.hooks:session_end" +thirdeye-claude-post-tool-use-failure = "thirdeye.platforms.claude.hooks:post_tool_use_failure" +thirdeye-claude-subagent-start = "thirdeye.platforms.claude.hooks:subagent_start" +thirdeye-claude-user-prompt-expansion = "thirdeye.platforms.claude.hooks:user_prompt_expansion" +thirdeye-claude-pre-compact = "thirdeye.platforms.claude.hooks:pre_compact" +thirdeye-claude-post-compact = "thirdeye.platforms.claude.hooks:post_compact" +thirdeye-claude-permission-denied = "thirdeye.platforms.claude.hooks:permission_denied" thirdeye-codex-notify = "thirdeye.platforms.codex.hooks:notify" [build-system] diff --git a/src/thirdeye/platforms/claude/constants.py b/src/thirdeye/platforms/claude/constants.py index 299c9fe..c63b665 100644 --- a/src/thirdeye/platforms/claude/constants.py +++ b/src/thirdeye/platforms/claude/constants.py @@ -17,4 +17,10 @@ "Notification": "thirdeye-claude-notification", "PermissionRequest": "thirdeye-claude-permission-request", "SessionEnd": "thirdeye-claude-session-end", + "PostToolUseFailure": "thirdeye-claude-post-tool-use-failure", + "SubagentStart": "thirdeye-claude-subagent-start", + "UserPromptExpansion": "thirdeye-claude-user-prompt-expansion", + "PreCompact": "thirdeye-claude-pre-compact", + "PostCompact": "thirdeye-claude-post-compact", + "PermissionDenied": "thirdeye-claude-permission-denied", } diff --git a/src/thirdeye/platforms/claude/hooks.py b/src/thirdeye/platforms/claude/hooks.py index 9682be4..04e80bc 100644 --- a/src/thirdeye/platforms/claude/hooks.py +++ b/src/thirdeye/platforms/claude/hooks.py @@ -136,6 +136,10 @@ def stop() -> None: def subagent_stop() -> None: + # SubagentStop keeps its historical "subagent_message" type on purpose: + # renaming it would orphan the 966 sessions already recorded under it. + # SubagentStart (below) uses a distinct "subagent_start" type; the + # start/stop asymmetry is intentional, not an oversight. _emit("subagent_message", _read_stdin()) @@ -155,3 +159,30 @@ def session_end() -> None: payload = _read_stdin() if _emit("session_end", payload) is not None: Store(Config.load()).close_session(payload["session_id"], platform=_PLATFORM) + + +def post_tool_use_failure() -> None: + # Emits "tool_result", not "error", on purpose: web/routes/sessions.py pairs + # each "tool_call" with a "tool_result", so a distinct type would leave failed + # tool calls rendering as dangling. The failure is evident from the payload. + _emit("tool_result", _read_stdin()) + + +def subagent_start() -> None: + _emit("subagent_start", _read_stdin()) + + +def user_prompt_expansion() -> None: + _emit("user_prompt_expansion", _read_stdin()) + + +def pre_compact() -> None: + _emit("compact_start", _read_stdin()) + + +def post_compact() -> None: + _emit("compact_end", _read_stdin()) + + +def permission_denied() -> None: + _emit("permission_denied", _read_stdin()) diff --git a/tests/test_claude_constants.py b/tests/test_claude_constants.py index 9e2e30b..16f19de 100644 --- a/tests/test_claude_constants.py +++ b/tests/test_claude_constants.py @@ -1,5 +1,8 @@ from __future__ import annotations +import tomllib +from pathlib import Path + from thirdeye.platforms.claude.constants import ( DISPLAY_NAME, HOOK_EVENTS, @@ -7,6 +10,8 @@ SETTINGS_FILE, ) +_PYPROJECT = Path(__file__).resolve().parent.parent / "pyproject.toml" + def test_platform_name(): assert PLATFORM_NAME == "claude" @@ -34,10 +39,29 @@ def test_hook_events_covers_known_lifecycle(): "Notification", "PermissionRequest", "SessionEnd", + "PostToolUseFailure", + "SubagentStart", + "UserPromptExpansion", + "PreCompact", + "PostCompact", + "PermissionDenied", } assert set(HOOK_EVENTS.keys()) == expected +def test_hook_events_has_sixteen_entries(): + assert len(HOOK_EVENTS) == 16 + + +def test_new_hook_events_have_expected_script_names(): + assert HOOK_EVENTS["PostToolUseFailure"] == "thirdeye-claude-post-tool-use-failure" + assert HOOK_EVENTS["SubagentStart"] == "thirdeye-claude-subagent-start" + assert HOOK_EVENTS["UserPromptExpansion"] == "thirdeye-claude-user-prompt-expansion" + assert HOOK_EVENTS["PreCompact"] == "thirdeye-claude-pre-compact" + assert HOOK_EVENTS["PostCompact"] == "thirdeye-claude-post-compact" + assert HOOK_EVENTS["PermissionDenied"] == "thirdeye-claude-permission-denied" + + def test_hook_event_scripts_unique(): assert len(set(HOOK_EVENTS.values())) == len(HOOK_EVENTS) @@ -45,3 +69,13 @@ def test_hook_event_scripts_unique(): def test_hook_event_scripts_have_thirdeye_prefix(): for script in HOOK_EVENTS.values(): assert script.startswith("thirdeye-claude-") + + +def test_every_hook_event_has_project_scripts_entry(): + # A hook wired without a matching [project.scripts] entry point fails + # silently at runtime; parse pyproject.toml to catch it in CI instead. + with _PYPROJECT.open("rb") as fh: + pyproject = tomllib.load(fh) + scripts = pyproject["project"]["scripts"] + for script in HOOK_EVENTS.values(): + assert script in scripts, f"missing [project.scripts] entry for {script}" diff --git a/tests/test_claude_hooks.py b/tests/test_claude_hooks.py index d61cd40..ca6acd7 100644 --- a/tests/test_claude_hooks.py +++ b/tests/test_claude_hooks.py @@ -461,6 +461,67 @@ def test_appends_permission_request(self, monkeypatch, env: Path): assert events[1]["data"]["tool_name"] == "Bash" +# -- new hook events (Claude Code 2.1.195) ------------------------------------- + + +# (handler, mapped event type) +_NEW_HANDLERS = [ + ("post_tool_use_failure", "tool_result"), + ("subagent_start", "subagent_start"), + ("user_prompt_expansion", "user_prompt_expansion"), + ("pre_compact", "compact_start"), + ("post_compact", "compact_end"), + ("permission_denied", "permission_denied"), +] + + +class TestNewHooks: + @pytest.mark.parametrize("handler_name,event_type", _NEW_HANDLERS) + def test_appends_one_event_of_mapped_type( + self, monkeypatch, env: Path, handler_name: str, event_type: str + ): + _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p", "extra": 42}) + getattr(hooks, handler_name)() + events = list(Store(Config.load()).reader("s1").iter_events()) + assert len(events) == 1 + assert events[0]["t"] == event_type + assert events[0]["data"]["extra"] == 42 + + @pytest.mark.parametrize("handler_name,event_type", _NEW_HANDLERS) + def test_strips_routing_keys(self, monkeypatch, env: Path, handler_name: str, event_type: str): + _stdin( + monkeypatch, + { + "session_id": "s1", + "cwd": "/p", + "transcript_path": "/long/path.jsonl", + "agent_transcript_path": "/long/agent.jsonl", + "kept": "yes", + }, + ) + getattr(hooks, handler_name)() + data = list(Store(Config.load()).reader("s1").iter_events())[0].get("data", {}) + assert "session_id" not in data + assert "cwd" not in data + assert "transcript_path" not in data + assert "agent_transcript_path" not in data + assert data == {"kept": "yes"} + + @pytest.mark.parametrize("handler_name,event_type", _NEW_HANDLERS) + def test_missing_session_id_is_noop( + self, monkeypatch, env: Path, handler_name: str, event_type: str + ): + _stdin(monkeypatch, {"cwd": "/p"}) + getattr(hooks, handler_name)() + assert list(Store(Config.load()).list_sessions()) == [] + + def test_pre_compact_preserves_trigger(self, monkeypatch, env: Path): + _stdin(monkeypatch, {"session_id": "s1", "cwd": "/p", "trigger": "auto"}) + hooks.pre_compact() + data = list(Store(Config.load()).reader("s1").iter_events())[0].get("data", {}) + assert data["trigger"] == "auto" + + # -- session_end --------------------------------------------------------------- diff --git a/tests/test_claude_install.py b/tests/test_claude_install.py index 6599435..077af64 100644 --- a/tests/test_claude_install.py +++ b/tests/test_claude_install.py @@ -307,11 +307,11 @@ def test_hook_structure_matches_claude_schema(self, tmp_path: Path): assert set(h.keys()) == {"type", "command"} assert h["type"] == "command" - def test_all_ten_events_registered(self, tmp_path: Path): + def test_all_events_registered(self, tmp_path: Path): settings_file = tmp_path / "settings.json" ClaudePlatform(settings_file=settings_file).install() settings = json.loads(settings_file.read_text()) - assert len(settings["hooks"]) == 10 + assert len(settings["hooks"]) == len(HOOK_EVENTS) class TestResolveCommandAbsolutePath: From d24254110f976729b09723d8ad23d7a92ea92a39 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Wed, 5 Aug 2026 23:38:55 -0700 Subject: [PATCH 06/15] Rewrite Claude usage extractor for OTel GenAI conventions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Normalize cache-inclusive input tokens (input_tokens + cache_read + cache_creation), emit one row per source frame carrying a dedup key (message.id → requestId → uuid), and drop synthetic placeholder frames. - _extract_row now requires type=="assistant" with a non-empty message.usage; the stale flat frame["usage"] fallback is removed. - "" model frames (zero-token injected messages) are dropped. - Cache fields pass through as int|None — absent stays absent, never coerced to 0. - reasoning_output_tokens is always None (Anthropic reports no thinking token breakout). - Deleted the "transcript frame shape is unverified" first-capture warning; the shape is now verified against a real transcript. Deduplication of the repeated per-block frames stays out of the writer (usage/read.py owns it). Tests drive from the scrubbed real-data fixture and its expected.json companion. Co-Authored-By: Claude Opus 4.8 --- src/thirdeye/platforms/claude/usage.py | 88 +++-- tests/test_usage_claude.py | 487 +++++++++++-------------- 2 files changed, 266 insertions(+), 309 deletions(-) diff --git a/src/thirdeye/platforms/claude/usage.py b/src/thirdeye/platforms/claude/usage.py index 6abfa94..e64c3b7 100644 --- a/src/thirdeye/platforms/claude/usage.py +++ b/src/thirdeye/platforms/claude/usage.py @@ -22,6 +22,10 @@ def capture_usage_claude( Returns the number of rows appended. Wrapped in @safe_capture so any error is logged to usage-errors.jsonl and the function returns None instead of raising. + + One row is appended per assistant frame - Claude writes one frame per content + block, all carrying the identical ``message.usage``. Collapsing those + duplicates is ``usage/read.py``'s job, never this writer's. """ if not transcript_path: return 0 @@ -40,20 +44,6 @@ def capture_usage_claude( sd = session_dir(thirdeye_home, "claude", session_id) store = UsageStore(sd) state = store.read_state() - if "transcript_offset" not in state: - log_capture_error( - thirdeye_home=thirdeye_home, - phase="discover_transcript", - message=( - "first capture for session; transcript frame shape is unverified " - "against a real Claude Code transcript — remove this warning once " - "format is confirmed end-to-end" - ), - platform="claude", - session_id=session_id, - source_path=str(transcript_path), - level="warn", - ) offset = int(state.get("transcript_offset", 0)) new_rows: list[UsageRow] = [] @@ -82,42 +72,66 @@ def capture_usage_claude( def _extract_row(frame: dict, session_id: str, triggering_seq: int) -> UsageRow | None: - """Return a UsageRow if `frame` looks like an assistant turn with usage. + """Return a UsageRow for an assistant frame carrying real API usage, else None. - Handles both nested (`frame["message"]["usage"]`) and flat - (`frame["usage"]`) shapes. + Emits a row per source frame; deduplication happens on read. Cache-inclusive + input tokens follow the OTel GenAI convention: ``gen_ai.usage.input_tokens`` + includes cache reads and cache creation, which Anthropic reports separately. """ if not isinstance(frame, dict): return None + if frame.get("type") != "assistant": + return None - message = frame.get("message") if isinstance(frame.get("message"), dict) else None - if message and "usage" in message: - model = message.get("model") or frame.get("model") - usage = message.get("usage") or {} - ts = message.get("timestamp") or frame.get("timestamp") or "" - elif "usage" in frame: - model = frame.get("model") - usage = frame.get("usage") or {} - ts = frame.get("timestamp") or "" - else: + message = frame.get("message") + if not isinstance(message, dict): + return None + usage = message.get("usage") + if not isinstance(usage, dict) or not usage: return None - if not isinstance(usage, dict): + # "" is Claude's zero-token placeholder for injected messages, + # not a real API call. + model = message.get("model") + if not model or model == "": return None - input_tokens = usage.get("input_tokens") - output_tokens = usage.get("output_tokens") - if input_tokens is None or output_tokens is None or not model: + + # Claude carries the timestamp at the top level, never inside `message`. + ts = frame.get("timestamp") or "" + + # message.id is the most reliable key (no nulls over a real transcript); + # fall back to requestId, then the frame uuid. + call_id = message.get("id") or frame.get("requestId") or frame.get("uuid") + if not call_id: + return None + + # Anthropic reports input_tokens EXCLUDING cache; add both cache classes back + # in for a comparable, cache-inclusive total. + raw_input = int(usage.get("input_tokens") or 0) + cache_read = usage.get("cache_read_input_tokens") + cache_crea = usage.get("cache_creation_input_tokens") + output = usage.get("output_tokens") + + # Absent both token fields means this frame carries no usage worth recording. + if usage.get("input_tokens") is None and output is None: return None - total = int(input_tokens) + int(output_tokens) + input_tokens = raw_input + int(cache_read or 0) + int(cache_crea or 0) return UsageRow( session_id=session_id, seq=triggering_seq, - ts=str(ts) if ts else "", + call_id=str(call_id), + ts=str(ts), platform="claude", - model=str(model), - input_tokens=int(input_tokens), - output_tokens=int(output_tokens), - total_tokens=total, + provider_name="anthropic", + response_model=str(model), + input_tokens=input_tokens, + output_tokens=int(output or 0), + operation_name="chat", + # Absent vs zero: pass cache fields through only when reported. + cache_read_input_tokens=int(cache_read) if cache_read is not None else None, + cache_creation_input_tokens=(int(cache_crea) if cache_crea is not None else None), + # Anthropic does not break out thinking/reasoning tokens. + reasoning_output_tokens=None, ) diff --git a/tests/test_usage_claude.py b/tests/test_usage_claude.py index fb85567..5c73f51 100644 --- a/tests/test_usage_claude.py +++ b/tests/test_usage_claude.py @@ -11,190 +11,199 @@ usage_log_path, usage_state_path, ) -from thirdeye.platforms.claude.usage import capture_usage_claude +from thirdeye.platforms.claude.usage import _extract_row, capture_usage_claude FIXTURE = Path(__file__).parent / "fixtures" / "usage" / "claude_transcript.jsonl" +EXPECTED_JSON = Path(__file__).parent / "fixtures" / "usage" / "claude_transcript.expected.json" -def test_capture_creates_rows_from_fixture(tmp_path: Path) -> None: - rows = capture_usage_claude( - thirdeye_home=tmp_path, - session_id="abc123", - transcript_path=str(FIXTURE), - triggering_seq=5, - ) - assert rows == 2 - jsonl = usage_jsonl_path(session_dir(tmp_path, "claude", "abc123")) - lines = jsonl.read_text().strip().splitlines() - assert len(lines) == 2 - first = json.loads(lines[0]) - assert first["platform"] == "claude" - assert first["seq"] == 5 - assert first["model"] - assert first["input_tokens"] == 12450 - assert first["output_tokens"] == 187 - assert first["total_tokens"] == 12637 +@pytest.fixture(scope="session") +def expected() -> dict: + return json.loads(EXPECTED_JSON.read_text()) -def test_capture_is_incremental(tmp_path: Path) -> None: - """Re-running against the same transcript at the saved offset produces 0 new rows.""" +def _capture_rows(tmp_path: Path, session_id: str = "abc123", seq: int = 5) -> list[dict]: + """Run a capture over the shipped fixture and return the parsed sidecar rows.""" capture_usage_claude( thirdeye_home=tmp_path, - session_id="abc", + session_id=session_id, transcript_path=str(FIXTURE), - triggering_seq=1, + triggering_seq=seq, ) - sd = session_dir(tmp_path, "claude", "abc") - state = json.loads(usage_state_path(sd).read_text()) - initial_offset = state["transcript_offset"] - assert initial_offset > 0 + jsonl = usage_jsonl_path(session_dir(tmp_path, "claude", session_id)) + return [json.loads(line) for line in jsonl.read_text().strip().splitlines()] - rows = capture_usage_claude( - thirdeye_home=tmp_path, - session_id="abc", - transcript_path=str(FIXTURE), - triggering_seq=2, - ) - assert rows == 0 - state2 = json.loads(usage_state_path(sd).read_text()) - assert state2["transcript_offset"] == initial_offset +def test_row_count_equals_assistant_minus_synthetic(tmp_path: Path, expected: dict) -> None: + """One row per source frame: every assistant frame except the synthetic placeholder. -def test_capture_missing_transcript_logs_error(tmp_path: Path) -> None: - rows = capture_usage_claude( - thirdeye_home=tmp_path, - session_id="abc", - transcript_path="/nonexistent/path.jsonl", - triggering_seq=1, - ) - assert rows == 0 - log = usage_log_path(tmp_path) - assert log.exists() and "open_source" in log.read_text() + This is the row count, NOT the de-duplicated call count. Collapsing the + repeated-message.id frames is usage/read.py's job, so the writer emits every + frame it sees. + """ + rows = _capture_rows(tmp_path) + assert len(rows) == expected["assistant_frames"] - expected["synthetic_frames"] -def test_capture_with_no_transcript_path(tmp_path: Path) -> None: - rows = capture_usage_claude( - thirdeye_home=tmp_path, - session_id="abc", - transcript_path=None, - triggering_seq=1, - ) - assert rows == 0 - sd = session_dir(tmp_path, "claude", "abc") - assert not usage_jsonl_path(sd).exists() +def test_distinct_call_id_equals_expected_calls(tmp_path: Path, expected: dict) -> None: + rows = _capture_rows(tmp_path) + distinct = {r["call_id"] for r in rows} + assert len(distinct) == expected["expected_calls"] + +def test_repeated_message_id_frames_share_identical_usage(tmp_path: Path, expected: dict) -> None: + rows = _capture_rows(tmp_path) + sample = expected["sample_call"] + repeated = [r for r in rows if r["call_id"] == expected["repeated_message_id"]] -def test_capture_handles_corrupt_jsonl_lines(tmp_path: Path) -> None: - transcript = tmp_path / "bad.jsonl" - transcript.write_text( - '{"type":"assistant","message":{"model":"claude-3","usage":{"input_tokens":10,"output_tokens":5}}}\n' - "this is not json\n" - '{"type":"assistant","message":{"model":"claude-3","usage":{"input_tokens":7,"output_tokens":3}}}\n' + assert len(repeated) == expected["repeated_message_id_frame_count"] + # Every frame of the repeated call carries the same id and token values. + assert all(r["call_id"] == expected["repeated_message_id"] for r in repeated) + first = repeated[0] + assert all( + r["gen_ai.usage.input_tokens"] == first["gen_ai.usage.input_tokens"] for r in repeated ) - rows = capture_usage_claude( - thirdeye_home=tmp_path, - session_id="abc", - transcript_path=str(transcript), - triggering_seq=10, + assert all( + r["gen_ai.usage.output_tokens"] == first["gen_ai.usage.output_tokens"] for r in repeated + ) + assert first["gen_ai.usage.input_tokens"] == sample["expected_inclusive_input_tokens"] + assert first["gen_ai.usage.output_tokens"] == sample["output_tokens"] + assert first["gen_ai.usage.cache_read.input_tokens"] == sample["cache_read_input_tokens"] + assert ( + first["gen_ai.usage.cache_creation.input_tokens"] == sample["cache_creation_input_tokens"] ) - assert rows == 2 -def test_capture_skips_non_assistant_frames(tmp_path: Path) -> None: - transcript = tmp_path / "mixed.jsonl" - transcript.write_text( - '{"type":"user","message":{"role":"user","content":"hi"}}\n' - '{"type":"assistant","message":{"model":"claude-3","usage":{"input_tokens":10,"output_tokens":5}}}\n' - '{"type":"meta"}\n' - ) - rows = capture_usage_claude( - thirdeye_home=tmp_path, - session_id="abc", - transcript_path=str(transcript), - triggering_seq=1, - ) - assert rows == 1 +def test_no_synthetic_model_row(tmp_path: Path) -> None: + rows = _capture_rows(tmp_path) + assert all(r["gen_ai.response.model"] != "" for r in rows) -def test_safe_capture_swallows_unexpected_error( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - import thirdeye.platforms.claude.usage as mod +def test_input_tokens_are_cache_inclusive(tmp_path: Path) -> None: + """gen_ai.usage.input_tokens must be >= reported cache reads + cache creation.""" + rows = _capture_rows(tmp_path) + for r in rows: + cache_read = r.get("gen_ai.usage.cache_read.input_tokens") or 0 + cache_crea = r.get("gen_ai.usage.cache_creation.input_tokens") or 0 + assert r["gen_ai.usage.input_tokens"] >= cache_read + cache_crea - monkeypatch.setattr( - mod, "_extract_row", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("oops")) - ) - transcript = tmp_path / "t.jsonl" - transcript.write_text( - '{"type":"assistant","message":{"model":"c","usage":{"input_tokens":1,"output_tokens":1}}}\n' - ) - result = capture_usage_claude( - thirdeye_home=tmp_path, - session_id="abc", - transcript_path=str(transcript), - triggering_seq=1, - ) - assert result is None - assert "RuntimeError" in usage_log_path(tmp_path).read_text() +def test_reasoning_output_tokens_always_none(tmp_path: Path) -> None: + """Anthropic does not break out thinking tokens, so the key is always omitted.""" + rows = _capture_rows(tmp_path) + assert all("gen_ai.usage.reasoning.output_tokens" not in r for r in rows) -def test_capture_flat_shape(tmp_path: Path) -> None: - """A frame without `message` wrapper but with `usage`/`model` at root is captured.""" - transcript = tmp_path / "flat.jsonl" - transcript.write_text( - '{"model":"claude-haiku","usage":{"input_tokens":50,"output_tokens":25},"timestamp":"2026-05-15T01:00:00Z"}\n' - ) - rows = capture_usage_claude( - thirdeye_home=tmp_path, - session_id="abc", - transcript_path=str(transcript), - triggering_seq=7, - ) - assert rows == 1 - sd = session_dir(tmp_path, "claude", "abc") - line = json.loads(usage_jsonl_path(sd).read_text().strip().splitlines()[0]) - assert line["model"] == "claude-haiku" - assert line["input_tokens"] == 50 - assert line["output_tokens"] == 25 - assert line["total_tokens"] == 75 - assert line["ts"] == "2026-05-15T01:00:00Z" - - -def test_capture_skips_frame_with_missing_model(tmp_path: Path) -> None: - transcript = tmp_path / "nomodel.jsonl" - transcript.write_text( - '{"type":"assistant","message":{"usage":{"input_tokens":10,"output_tokens":5}}}\n' - ) - rows = capture_usage_claude( - thirdeye_home=tmp_path, - session_id="abc", - transcript_path=str(transcript), - triggering_seq=1, + +def test_provider_and_operation_metadata(tmp_path: Path) -> None: + rows = _capture_rows(tmp_path) + assert all(r["gen_ai.provider.name"] == "anthropic" for r in rows) + assert all(r["gen_ai.operation.name"] == "chat" for r in rows) + assert all(r["platform"] == "claude" for r in rows) + assert all(r["seq"] == 5 for r in rows) + + +# --- absent vs zero ------------------------------------------------------- + + +def test_absent_cache_read_yields_none() -> None: + frame = { + "type": "assistant", + "timestamp": "2026-07-19T00:00:00Z", + "message": { + "id": "msg_absent", + "model": "claude-sonnet-5", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }, + } + row = _extract_row(frame, "sid", 1) + assert row is not None + assert row.cache_read_input_tokens is None + assert row.cache_creation_input_tokens is None + # input stays raw when no cache is reported. + assert row.input_tokens == 10 + + +def test_explicit_zero_cache_read_yields_zero() -> None: + frame = { + "type": "assistant", + "timestamp": "2026-07-19T00:00:00Z", + "message": { + "id": "msg_zero", + "model": "claude-sonnet-5", + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "cache_read_input_tokens": 0, + }, + }, + } + row = _extract_row(frame, "sid", 1) + assert row is not None + assert row.cache_read_input_tokens == 0 + + +# --- call_id fallback chain ---------------------------------------------- + + +def _frame(message: dict, **top: object) -> dict: + base = {"type": "assistant", "timestamp": "2026-07-19T00:00:00Z", "message": message} + base.update(top) + return base + + +def test_call_id_prefers_message_id() -> None: + frame = _frame( + {"id": "msg_x", "model": "m", "usage": {"input_tokens": 1, "output_tokens": 1}}, + requestId="req_x", + uuid="uuid_x", ) - assert rows == 0 + row = _extract_row(frame, "sid", 1) + assert row is not None and row.call_id == "msg_x" -def test_capture_skips_frame_with_missing_token_field(tmp_path: Path) -> None: - transcript = tmp_path / "partial.jsonl" - transcript.write_text( - '{"type":"assistant","message":{"model":"c","usage":{"input_tokens":10}}}\n' - '{"type":"assistant","message":{"model":"c","usage":{"output_tokens":5}}}\n' +def test_call_id_falls_back_to_request_id() -> None: + frame = _frame( + {"model": "m", "usage": {"input_tokens": 1, "output_tokens": 1}}, + requestId="req_x", + uuid="uuid_x", ) - rows = capture_usage_claude( - thirdeye_home=tmp_path, - session_id="abc", - transcript_path=str(transcript), - triggering_seq=1, + row = _extract_row(frame, "sid", 1) + assert row is not None and row.call_id == "req_x" + + +def test_call_id_falls_back_to_uuid() -> None: + frame = _frame( + {"model": "m", "usage": {"input_tokens": 1, "output_tokens": 1}}, + uuid="uuid_x", ) - assert rows == 0 + row = _extract_row(frame, "sid", 1) + assert row is not None and row.call_id == "uuid_x" + + +def test_call_id_all_missing_returns_none() -> None: + frame = _frame({"model": "m", "usage": {"input_tokens": 1, "output_tokens": 1}}) + assert _extract_row(frame, "sid", 1) is None + + +# --- offset / incremental behaviour -------------------------------------- def test_capture_appends_only_new_rows_on_growth(tmp_path: Path) -> None: - """When transcript grows between calls, only the appended portion is processed.""" transcript = tmp_path / "grow.jsonl" - transcript.write_text( - '{"type":"assistant","message":{"model":"c","usage":{"input_tokens":10,"output_tokens":5}}}\n' + frame_a = json.dumps( + { + "type": "assistant", + "timestamp": "2026-07-19T00:00:00Z", + "requestId": "req_a", + "message": { + "id": "msg_a", + "model": "c", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }, + } ) + transcript.write_text(frame_a + "\n") + rows1 = capture_usage_claude( thirdeye_home=tmp_path, session_id="abc", @@ -203,11 +212,20 @@ def test_capture_appends_only_new_rows_on_growth(tmp_path: Path) -> None: ) assert rows1 == 1 + frame_b = json.dumps( + { + "type": "assistant", + "timestamp": "2026-07-19T00:01:00Z", + "requestId": "req_b", + "message": { + "id": "msg_b", + "model": "c", + "usage": {"input_tokens": 20, "output_tokens": 7}, + }, + } + ) with transcript.open("a") as f: - f.write( - '{"type":"assistant","message":{"model":"c","usage":{"input_tokens":20,"output_tokens":7}}}\n' - '{"type":"assistant","message":{"model":"c","usage":{"input_tokens":30,"output_tokens":3}}}\n' - ) + f.write(frame_b + "\n") rows2 = capture_usage_claude( thirdeye_home=tmp_path, @@ -215,148 +233,73 @@ def test_capture_appends_only_new_rows_on_growth(tmp_path: Path) -> None: transcript_path=str(transcript), triggering_seq=2, ) - assert rows2 == 2 + assert rows2 == 1 sd = session_dir(tmp_path, "claude", "abc") lines = usage_jsonl_path(sd).read_text().strip().splitlines() - assert len(lines) == 3 + assert len(lines) == 2 second = json.loads(lines[1]) - third = json.loads(lines[2]) - assert second["seq"] == 2 and second["input_tokens"] == 20 - assert third["seq"] == 2 and third["input_tokens"] == 30 + assert second["seq"] == 2 and second["call_id"] == "msg_b" -def test_capture_advances_offset_with_no_rows(tmp_path: Path) -> None: - """Even when no assistant frames are found, the offset must advance past the read bytes.""" - transcript = tmp_path / "user_only.jsonl" - transcript.write_text( - '{"type":"user","message":{"role":"user","content":"hi"}}\n{"type":"meta"}\n' - ) - rows = capture_usage_claude( +def test_capture_is_incremental(tmp_path: Path) -> None: + capture_usage_claude( thirdeye_home=tmp_path, session_id="abc", - transcript_path=str(transcript), - triggering_seq=3, + transcript_path=str(FIXTURE), + triggering_seq=1, ) - assert rows == 0 sd = session_dir(tmp_path, "claude", "abc") state = json.loads(usage_state_path(sd).read_text()) - assert state["transcript_offset"] == transcript.stat().st_size - # last_seq should remain at the default (-1) since no rows were appended. - assert state["last_seq"] == -1 - + initial_offset = state["transcript_offset"] + assert initial_offset > 0 -def test_capture_updates_last_seq_when_rows_appended(tmp_path: Path) -> None: - transcript = tmp_path / "t.jsonl" - transcript.write_text( - '{"type":"assistant","message":{"model":"c","usage":{"input_tokens":1,"output_tokens":1}}}\n' - ) - capture_usage_claude( + rows = capture_usage_claude( thirdeye_home=tmp_path, session_id="abc", - transcript_path=str(transcript), - triggering_seq=42, + transcript_path=str(FIXTURE), + triggering_seq=2, ) - sd = session_dir(tmp_path, "claude", "abc") - state = json.loads(usage_state_path(sd).read_text()) - assert state["last_seq"] == 42 + assert rows == 0 + state2 = json.loads(usage_state_path(sd).read_text()) + assert state2["transcript_offset"] == initial_offset -def test_stop_hook_invokes_capture_and_survives_failure( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """The stop hook must call capture_usage_claude with the unstripped transcript_path - and must not raise even if capture errors internally.""" - from thirdeye.config import Config - from thirdeye.platforms.claude import hooks - from thirdeye.platforms.claude import usage as usage_mod +# --- error handling / logging -------------------------------------------- - home = tmp_path / "thirdeye" - home.mkdir() - monkeypatch.setattr(Config, "load", classmethod(lambda cls: Config(root=home))) - transcript = tmp_path / "t.jsonl" - transcript.write_text( - '{"type":"assistant","message":{"model":"claude-3","usage":{"input_tokens":4,"output_tokens":2}}}\n' +def test_capture_missing_transcript_logs_error(tmp_path: Path) -> None: + rows = capture_usage_claude( + thirdeye_home=tmp_path, + session_id="abc", + transcript_path="/nonexistent/path.jsonl", + triggering_seq=1, ) + assert rows == 0 + log = usage_log_path(tmp_path) + assert log.exists() and "open_source" in log.read_text() - payload = { - "session_id": "hooksid", - "cwd": str(tmp_path), - "transcript_path": str(transcript), - "extra": "kept-in-event", - } - monkeypatch.setattr(hooks, "_read_stdin", lambda: payload) - - captured: dict = {} - original = usage_mod.capture_usage_claude - - def spy(**kwargs): - captured.update(kwargs) - return original(**kwargs) - monkeypatch.setattr( - hooks, - "_strip_payload", - lambda p: {k: v for k, v in p.items() if k not in {"session_id", "cwd", "transcript_path"}}, +def test_capture_with_no_transcript_path(tmp_path: Path) -> None: + rows = capture_usage_claude( + thirdeye_home=tmp_path, + session_id="abc", + transcript_path=None, + triggering_seq=1, ) - # Patch where the function is looked up: it's imported inside `stop`. - monkeypatch.setattr("thirdeye.platforms.claude.usage.capture_usage_claude", spy) - - hooks.stop() - - assert captured["session_id"] == "hooksid" - assert captured["transcript_path"] == str(transcript) - assert isinstance(captured["triggering_seq"], int) - # Capture actually wrote a sidecar row - sd = session_dir(home, "claude", "hooksid") - assert usage_jsonl_path(sd).exists() - - -def test_stop_hook_with_no_session_id_does_nothing( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - from thirdeye.platforms.claude import hooks - - monkeypatch.setattr(hooks, "_read_stdin", lambda: {}) - called = {"count": 0} - - def fake(**kwargs): - called["count"] += 1 - - monkeypatch.setattr("thirdeye.platforms.claude.usage.capture_usage_claude", fake) - # Should return without touching capture or raising - hooks.stop() - assert called["count"] == 0 - - -def test_stop_hook_survives_capture_raising( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """If capture_usage_claude were ever to raise, the stop hook should still exit cleanly. - (In production it's @safe_capture wrapped, so it can't — but the hook itself shouldn't - add another try/except, and this guards that the wrapper is actually in place.)""" - from thirdeye.config import Config - from thirdeye.platforms.claude import hooks - from thirdeye.platforms.claude import usage as usage_mod - - home = tmp_path / "thirdeye" - home.mkdir() - monkeypatch.setattr(Config, "load", classmethod(lambda cls: Config(root=home))) - - payload = { - "session_id": "boom", - "cwd": str(tmp_path), - "transcript_path": "/does/not/exist.jsonl", - } - monkeypatch.setattr(hooks, "_read_stdin", lambda: payload) + assert rows == 0 + sd = session_dir(tmp_path, "claude", "abc") + assert not usage_jsonl_path(sd).exists() - # Force the inner extractor to blow up; @safe_capture should swallow it. - monkeypatch.setattr( - usage_mod, - "_extract_row", - lambda *a, **k: (_ for _ in ()).throw(RuntimeError("kaboom")), - ) - # Must not raise - hooks.stop() +def test_first_capture_emits_no_unverified_warning(tmp_path: Path) -> None: + """The old 'shape is unverified' warning was deleted; first capture logs nothing.""" + capture_usage_claude( + thirdeye_home=tmp_path, + session_id="fresh", + transcript_path=str(FIXTURE), + triggering_seq=1, + ) + log = usage_log_path(tmp_path) + contents = log.read_text() if log.exists() else "" + assert "unverified" not in contents From 2f3a56adc2a07b6798ac355a3e2c121fd0620394 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Wed, 5 Aug 2026 23:42:29 -0700 Subject: [PATCH 07/15] Add edge-case coverage for Claude usage extractor Cover the spec-mandated rejection branches (empty usage, both token fields absent, non-assistant/non-dict frames, synthetic model) and the corrupt-line skip, raising _extract_row coverage to 98%. Co-Authored-By: Claude Opus 4.8 --- tests/test_usage_claude.py | 84 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/tests/test_usage_claude.py b/tests/test_usage_claude.py index 5c73f51..e301b31 100644 --- a/tests/test_usage_claude.py +++ b/tests/test_usage_claude.py @@ -303,3 +303,87 @@ def test_first_capture_emits_no_unverified_warning(tmp_path: Path) -> None: log = usage_log_path(tmp_path) contents = log.read_text() if log.exists() else "" assert "unverified" not in contents + + +# --- rejected-frame branches (spec steps 1 & 5) -------------------------- + + +def test_non_dict_frame_yields_none() -> None: + assert _extract_row("not a dict", "sid", 1) is None # type: ignore[arg-type] + + +def test_non_dict_message_yields_none() -> None: + frame = {"type": "assistant", "message": "not a dict"} + assert _extract_row(frame, "sid", 1) is None + + +def test_non_assistant_frame_yields_none() -> None: + """Only type=="assistant" frames are candidates; a user frame is dropped.""" + frame = { + "type": "user", + "message": {"id": "msg_u", "usage": {"input_tokens": 5, "output_tokens": 2}}, + } + assert _extract_row(frame, "sid", 1) is None + + +def test_empty_usage_dict_yields_none() -> None: + """An assistant frame whose message.usage is an empty dict carries no usage.""" + frame = _frame({"id": "msg_e", "model": "m", "usage": {}}) + assert _extract_row(frame, "sid", 1) is None + + +def test_missing_usage_key_yields_none() -> None: + frame = _frame({"id": "msg_n", "model": "m"}) + assert _extract_row(frame, "sid", 1) is None + + +def test_both_token_fields_absent_yields_none() -> None: + """Spec step 5: with neither input_tokens nor output_tokens reported, drop it. + + Cache-only usage (no primary token counts) is not a recordable call. + """ + frame = _frame( + {"id": "msg_c", "model": "m", "usage": {"cache_read_input_tokens": 100}}, + ) + assert _extract_row(frame, "sid", 1) is None + + +def test_output_absent_but_input_present_yields_row_with_zero_output() -> None: + """Only one primary field present is still a call; the missing one reads as 0.""" + frame = _frame({"id": "msg_o", "model": "m", "usage": {"input_tokens": 12}}) + row = _extract_row(frame, "sid", 1) + assert row is not None + assert row.input_tokens == 12 + assert row.output_tokens == 0 + + +def test_synthetic_model_frame_yields_none() -> None: + frame = _frame( + {"id": "msg_s", "model": "", "usage": {"input_tokens": 0, "output_tokens": 0}}, + ) + assert _extract_row(frame, "sid", 1) is None + + +def test_capture_skips_corrupt_jsonl_lines(tmp_path: Path) -> None: + """A malformed line between valid frames is skipped, not fatal.""" + transcript = tmp_path / "corrupt.jsonl" + good = json.dumps( + { + "type": "assistant", + "timestamp": "2026-07-19T00:00:00Z", + "requestId": "req_g", + "message": { + "id": "msg_g", + "model": "c", + "usage": {"input_tokens": 3, "output_tokens": 1}, + }, + } + ) + transcript.write_text(good + "\nnot json at all\n" + good + "\n") + rows = capture_usage_claude( + thirdeye_home=tmp_path, + session_id="abc", + transcript_path=str(transcript), + triggering_seq=1, + ) + assert rows == 2 From e0548d836d17b51649a64369fab898d88f1b850c Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Wed, 5 Aug 2026 23:47:09 -0700 Subject: [PATCH 08/15] Move daily aggregation onto deduplicated read path Replace UsageStore(sd).iter_rows() with iter_calls(sd) in aggregate_by_day so duplicated per-frame rows collapse to one call before bucketing. TimeBucket keeps its four fields plus derived total_tokens. Rewrite tests against the new UsageRow signature, guarding dedup with the six-duplicate case. Co-Authored-By: Claude Opus 4.8 --- src/thirdeye/usage/aggregate.py | 4 +- tests/test_usage_aggregate.py | 240 ++++++++++++-------------------- 2 files changed, 94 insertions(+), 150 deletions(-) diff --git a/src/thirdeye/usage/aggregate.py b/src/thirdeye/usage/aggregate.py index eada779..683889e 100644 --- a/src/thirdeye/usage/aggregate.py +++ b/src/thirdeye/usage/aggregate.py @@ -9,7 +9,7 @@ from thirdeye.config import Config from thirdeye.paths import session_dir as _session_dir from thirdeye.store import Store -from thirdeye.usage.store import UsageStore +from thirdeye.usage.read import iter_calls @dataclass(frozen=True) @@ -56,7 +56,7 @@ def aggregate_by_day( store = Store(config) for meta in store.list_sessions(platform=platform, since=since, until=until): sd = _session_dir(config.root, meta.platform, meta.session_id) - for row in UsageStore(sd).iter_rows(): + for row in iter_calls(sd): day = _row_day(row.ts) b = sums[day] b["events"] = int(b["events"]) + 1 diff --git a/tests/test_usage_aggregate.py b/tests/test_usage_aggregate.py index 5a67f19..d83afbd 100644 --- a/tests/test_usage_aggregate.py +++ b/tests/test_usage_aggregate.py @@ -45,18 +45,26 @@ def _row( platform: str, seq: int, ts: str, + call_id: str | None = None, input_tokens: int = 100, output_tokens: int = 10, + cache_read_input_tokens: int | None = None, + cache_creation_input_tokens: int | None = None, + reasoning_output_tokens: int | None = None, ) -> UsageRow: return UsageRow( session_id=sid, seq=seq, + call_id=call_id if call_id is not None else f"{sid}-{seq}", ts=ts, platform=platform, - model="claude-opus-4-7", + provider_name="anthropic", + response_model="claude-opus-4-7", input_tokens=input_tokens, output_tokens=output_tokens, - total_tokens=input_tokens + output_tokens, + cache_read_input_tokens=cache_read_input_tokens, + cache_creation_input_tokens=cache_creation_input_tokens, + reasoning_output_tokens=reasoning_output_tokens, ) @@ -75,24 +83,49 @@ def test_empty_store_returns_empty_report(config: Config) -> None: assert report.totals.output_tokens == 0 -def test_two_sessions_two_days_aggregates(config: Config) -> None: - sd1 = _make_session( +def test_duplicate_call_id_counted_once(config: Config) -> None: + """Six duplicate frames of one call (100/10) collapse to a single call. + + This is the guard against dedup silently regressing: the bucket must be + 100 input / 10 output, not 600 / 60. + """ + sd = _make_session( config.root, sid="s1", platform="claude", started_at="2026-05-20T10:00:00.000Z", ) - UsageStore(sd1).append( + UsageStore(sd).append( [ - _row(sid="s1", platform="claude", seq=0, ts="2026-05-20T10:00:00.000Z"), - _row(sid="s1", platform="claude", seq=1, ts="2026-05-20T11:00:00.000Z"), + _row( + sid="s1", + platform="claude", + seq=0, + ts="2026-05-20T10:00:00.000Z", + call_id="call-abc", + ) + for _ in range(6) ] ) + + report = aggregate_by_day(config) + by_day = {b.day: b for b in report.buckets} + assert by_day["2026-05-20"].events == 1 + assert by_day["2026-05-20"].input_tokens == 100 + assert by_day["2026-05-20"].output_tokens == 10 + assert report.totals.input_tokens == 100 + assert report.totals.output_tokens == 10 + + +def test_two_sessions_same_day_sum_into_one_bucket(config: Config) -> None: + sd1 = _make_session( + config.root, sid="s1", platform="claude", started_at="2026-05-20T10:00:00.000Z" + ) + UsageStore(sd1).append( + [_row(sid="s1", platform="claude", seq=0, ts="2026-05-20T10:00:00.000Z")] + ) sd2 = _make_session( - config.root, - sid="s2", - platform="claude", - started_at="2026-05-21T10:00:00.000Z", + config.root, sid="s2", platform="claude", started_at="2026-05-20T12:00:00.000Z" ) UsageStore(sd2).append( [ @@ -100,63 +133,46 @@ def test_two_sessions_two_days_aggregates(config: Config) -> None: sid="s2", platform="claude", seq=0, - ts="2026-05-21T10:00:00.000Z", + ts="2026-05-20T12:00:00.000Z", input_tokens=50, output_tokens=5, - ), + ) ] ) report = aggregate_by_day(config) - assert [b.day for b in report.buckets] == ["2026-05-20", "2026-05-21"] + assert [b.day for b in report.buckets] == ["2026-05-20"] + bucket = report.buckets[0] + assert bucket.sessions == 2 + assert bucket.events == 2 + assert bucket.input_tokens == 150 + assert bucket.output_tokens == 15 - by_day = {b.day: b for b in report.buckets} - assert by_day["2026-05-20"].events == 2 - assert by_day["2026-05-20"].sessions == 1 - assert by_day["2026-05-20"].input_tokens == 200 - assert by_day["2026-05-20"].output_tokens == 20 - assert by_day["2026-05-21"].events == 1 - assert by_day["2026-05-21"].sessions == 1 - assert by_day["2026-05-21"].input_tokens == 50 - assert by_day["2026-05-21"].output_tokens == 5 - - assert report.totals.events == 3 - assert report.totals.sessions == 2 - assert report.totals.input_tokens == 250 - assert report.totals.output_tokens == 25 - assert report.totals.total_tokens == 275 - - -def test_platform_filter_excludes_others(config: Config) -> None: - sd_claude = _make_session( - config.root, sid="c1", platform="claude", started_at="2026-05-20T10:00:00.000Z" + +def test_rows_spanning_three_days_zero_fill(config: Config) -> None: + sd1 = _make_session( + config.root, sid="s1", platform="claude", started_at="2026-05-20T10:00:00.000Z" ) - UsageStore(sd_claude).append( - [_row(sid="c1", platform="claude", seq=0, ts="2026-05-20T10:00:00.000Z")] + UsageStore(sd1).append( + [_row(sid="s1", platform="claude", seq=0, ts="2026-05-20T10:00:00.000Z")] ) - sd_codex = _make_session( - config.root, sid="x1", platform="codex", started_at="2026-05-20T10:00:00.000Z" + sd2 = _make_session( + config.root, sid="s2", platform="claude", started_at="2026-05-22T10:00:00.000Z" ) - UsageStore(sd_codex).append( - [ - _row( - sid="x1", - platform="codex", - seq=0, - ts="2026-05-20T10:00:00.000Z", - input_tokens=999, - output_tokens=0, - ) - ] + UsageStore(sd2).append( + [_row(sid="s2", platform="claude", seq=0, ts="2026-05-22T10:00:00.000Z")] ) - report = aggregate_by_day(config, platform="claude") - assert report.totals.sessions == 1 - assert report.totals.input_tokens == 100 - assert report.totals.events == 1 + report = aggregate_by_day(config) + assert [b.day for b in report.buckets] == ["2026-05-20", "2026-05-21", "2026-05-22"] + by_day = {b.day: b for b in report.buckets} + assert by_day["2026-05-21"].events == 0 + assert by_day["2026-05-21"].sessions == 0 + assert by_day["2026-05-21"].input_tokens == 0 + assert by_day["2026-05-21"].output_tokens == 0 -def test_since_until_filter_at_session_level(config: Config) -> None: +def test_since_until_bounds_control_range(config: Config) -> None: sd_old = _make_session( config.root, sid="old", @@ -181,123 +197,51 @@ def test_since_until_filter_at_session_level(config: Config) -> None: since = datetime(2026, 5, 15, tzinfo=UTC) until = datetime(2026, 5, 25, tzinfo=UTC) report = aggregate_by_day(config, since=since, until=until) + days = {b.day for b in report.buckets} assert report.totals.sessions == 1 assert report.totals.events == 1 - assert "2026-05-20" in {b.day for b in report.buckets} - assert "2026-05-01" not in {b.day for b in report.buckets} + assert "2026-05-20" in days + assert "2026-05-01" not in days -def test_session_counted_distinctly_per_day(config: Config) -> None: +def test_z_suffixed_timestamp_buckets_to_utc_day(config: Config) -> None: sd = _make_session( config.root, sid="s1", platform="claude", - started_at="2026-05-20T10:00:00.000Z", - last_ts="2026-05-21T10:00:00.000Z", - ) - UsageStore(sd).append( - [ - _row(sid="s1", platform="claude", seq=0, ts="2026-05-20T10:00:00.000Z"), - _row(sid="s1", platform="claude", seq=1, ts="2026-05-21T10:00:00.000Z"), - ] - ) - - report = aggregate_by_day(config) - by_day = {b.day: b for b in report.buckets} - assert by_day["2026-05-20"].sessions == 1 - assert by_day["2026-05-21"].sessions == 1 - assert report.totals.sessions == 1 - - -def test_zero_day_buckets_between_seen_days(config: Config) -> None: - sd1 = _make_session( - config.root, sid="s1", platform="claude", started_at="2026-05-20T10:00:00.000Z" - ) - UsageStore(sd1).append( - [_row(sid="s1", platform="claude", seq=0, ts="2026-05-20T10:00:00.000Z")] - ) - sd2 = _make_session( - config.root, sid="s2", platform="claude", started_at="2026-05-23T10:00:00.000Z" - ) - UsageStore(sd2).append( - [_row(sid="s2", platform="claude", seq=0, ts="2026-05-23T10:00:00.000Z")] + started_at="2026-05-20T23:00:00.000Z", + last_ts="2026-05-20T23:30:00.000Z", ) + UsageStore(sd).append([_row(sid="s1", platform="claude", seq=0, ts="2026-05-20T23:30:00.000Z")]) report = aggregate_by_day(config) - assert [b.day for b in report.buckets] == [ - "2026-05-20", - "2026-05-21", - "2026-05-22", - "2026-05-23", - ] - by_day = {b.day: b for b in report.buckets} - assert by_day["2026-05-21"].events == 0 - assert by_day["2026-05-21"].sessions == 0 - assert by_day["2026-05-21"].input_tokens == 0 - assert by_day["2026-05-22"].events == 0 + by_day = {b.day: b.events for b in report.buckets} + assert by_day.get("2026-05-20") == 1 -def test_bounds_extend_range_when_wider_than_data(config: Config) -> None: +def test_none_cache_attributes_aggregate_without_raising(config: Config) -> None: sd = _make_session( config.root, sid="s1", platform="claude", started_at="2026-05-20T10:00:00.000Z" ) - UsageStore(sd).append([_row(sid="s1", platform="claude", seq=0, ts="2026-05-20T10:00:00.000Z")]) - - since = datetime(2026, 5, 18, tzinfo=UTC) - until = datetime(2026, 5, 22, tzinfo=UTC) - report = aggregate_by_day(config, since=since, until=until) - days = [b.day for b in report.buckets] - assert days[0] == "2026-05-18" - assert days[-1] == "2026-05-22" - assert len(days) == 5 - - -def test_row_day_handles_offset_timestamp(config: Config) -> None: - sd = _make_session( - config.root, - sid="s1", - platform="claude", - started_at="2026-05-20T23:00:00.000Z", - last_ts="2026-05-20T23:30:00.000Z", - ) UsageStore(sd).append( [ - UsageRow( - session_id="s1", - seq=0, - ts="2026-05-20T23:30:00.000+00:00", - platform="claude", - model="m", - input_tokens=1, - output_tokens=1, - total_tokens=2, - ), - UsageRow( - session_id="s1", - seq=1, - ts="2026-05-21T01:30:00.000+02:00", + _row( + sid="s1", platform="claude", - model="m", - input_tokens=2, - output_tokens=2, - total_tokens=4, - ), + seq=0, + ts="2026-05-20T10:00:00.000Z", + cache_read_input_tokens=None, + cache_creation_input_tokens=None, + reasoning_output_tokens=None, + ) ] ) report = aggregate_by_day(config) - by_day = {b.day: b.events for b in report.buckets} - assert by_day.get("2026-05-20") == 2 - - -def test_empty_with_bounds_produces_zero_buckets(config: Config) -> None: - since = datetime(2026, 5, 18, tzinfo=UTC) - until = datetime(2026, 5, 20, tzinfo=UTC) - report = aggregate_by_day(config, since=since, until=until) - days = [b.day for b in report.buckets] - assert days == ["2026-05-18", "2026-05-19", "2026-05-20"] - assert all(b.events == 0 and b.sessions == 0 for b in report.buckets) - assert report.totals.events == 0 + bucket = report.buckets[0] + assert bucket.input_tokens == 100 + assert bucket.output_tokens == 10 + assert bucket.total_tokens == 110 def test_aggregate_returns_immutable_dataclasses(config: Config) -> None: From 0ec51ef5c77e01d5e71578247c1ebb3502e2f6e3 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Wed, 5 Aug 2026 23:48:19 -0700 Subject: [PATCH 09/15] Rebuild usage index on OTel GenAI schema, key by call_id Bump SCHEMA_VERSION to 2 and replace the usage table: - PRIMARY KEY (session_id, call_id) instead of (session_id, seq), so a turn's shared Stop-hook seq no longer collapses every call into one row. - gen_ai_* column names; nullable cache/reasoning columns preserve the absent-vs-zero distinction. - seq demoted to an indexed plain column; indexes on response_model, ts, platform, seq. connect() drops and rebuilds when user_version < SCHEMA_VERSION (the DB is purely derived from sidecars, so no data migration is warranted). _refresh_one reads through iter_calls (the single dedup path) and upserts with ON CONFLICT DO UPDATE so a later append that corrects a call wins. Byte-offset bookmark still gates re-reads; shrunk-sidecar and malformed-line logging are preserved. Co-Authored-By: Claude Opus 4.8 --- src/thirdeye/usage/index.py | 184 ++++++++++++++++++++++++------------ 1 file changed, 121 insertions(+), 63 deletions(-) diff --git a/src/thirdeye/usage/index.py b/src/thirdeye/usage/index.py index 9b4c1e0..1a143a9 100644 --- a/src/thirdeye/usage/index.py +++ b/src/thirdeye/usage/index.py @@ -6,30 +6,63 @@ from thirdeye.paths import sessions_root, usage_db_path, usage_jsonl_path from thirdeye.usage.errlog import log_capture_error +from thirdeye.usage.read import iter_calls -SCHEMA_VERSION = 1 +# Bumped to 2 for the OTel GenAI schema. usage.db is purely derived from the +# sidecars, so a user_version below SCHEMA_VERSION drops and rebuilds rather +# than migrating. +SCHEMA_VERSION = 2 SCHEMA_SQL = """ CREATE TABLE IF NOT EXISTS usage ( - session_id TEXT NOT NULL, - seq INTEGER NOT NULL, - ts TEXT NOT NULL, - platform TEXT NOT NULL, - model TEXT NOT NULL, - input_tokens INTEGER NOT NULL, - output_tokens INTEGER NOT NULL, - total_tokens INTEGER NOT NULL, - PRIMARY KEY (session_id, seq) + session_id TEXT NOT NULL, + call_id TEXT NOT NULL, + seq INTEGER NOT NULL, + ts TEXT NOT NULL, + platform TEXT NOT NULL, + gen_ai_provider_name TEXT NOT NULL, + gen_ai_operation_name TEXT NOT NULL, + gen_ai_response_model TEXT NOT NULL, + gen_ai_usage_input_tokens INTEGER NOT NULL, + gen_ai_usage_output_tokens INTEGER NOT NULL, + gen_ai_usage_cache_read_input_tokens INTEGER, + gen_ai_usage_cache_creation_input_tokens INTEGER, + gen_ai_usage_reasoning_output_tokens INTEGER, + PRIMARY KEY (session_id, call_id) ); -CREATE INDEX IF NOT EXISTS idx_usage_model ON usage (model); +CREATE INDEX IF NOT EXISTS idx_usage_model ON usage (gen_ai_response_model); CREATE INDEX IF NOT EXISTS idx_usage_ts ON usage (ts); CREATE INDEX IF NOT EXISTS idx_usage_platform ON usage (platform); +CREATE INDEX IF NOT EXISTS idx_usage_seq ON usage (seq); CREATE TABLE IF NOT EXISTS usage_sync ( session_id TEXT PRIMARY KEY, last_jsonl_size INTEGER NOT NULL ); """ +# Column order for the usage table, mirrored by the upsert below. +_INSERT_SQL = """ +INSERT INTO usage ( + session_id, call_id, seq, ts, platform, + gen_ai_provider_name, gen_ai_operation_name, gen_ai_response_model, + gen_ai_usage_input_tokens, gen_ai_usage_output_tokens, + gen_ai_usage_cache_read_input_tokens, gen_ai_usage_cache_creation_input_tokens, + gen_ai_usage_reasoning_output_tokens +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT (session_id, call_id) DO UPDATE SET + seq = excluded.seq, + ts = excluded.ts, + platform = excluded.platform, + gen_ai_provider_name = excluded.gen_ai_provider_name, + gen_ai_operation_name = excluded.gen_ai_operation_name, + gen_ai_response_model = excluded.gen_ai_response_model, + gen_ai_usage_input_tokens = excluded.gen_ai_usage_input_tokens, + gen_ai_usage_output_tokens = excluded.gen_ai_usage_output_tokens, + gen_ai_usage_cache_read_input_tokens = excluded.gen_ai_usage_cache_read_input_tokens, + gen_ai_usage_cache_creation_input_tokens = excluded.gen_ai_usage_cache_creation_input_tokens, + gen_ai_usage_reasoning_output_tokens = excluded.gen_ai_usage_reasoning_output_tokens +""" + class UsageIndex: def __init__(self, thirdeye_home: Path) -> None: @@ -39,21 +72,26 @@ def __init__(self, thirdeye_home: Path) -> None: def connect(self) -> sqlite3.Connection: self.db_path.parent.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(self.db_path) - conn.executescript(SCHEMA_SQL) current = conn.execute("PRAGMA user_version").fetchone()[0] if current < SCHEMA_VERSION: + # usage.db is purely derived from the sidecars, so an out-of-date + # schema is dropped and rebuilt rather than migrated. + conn.executescript("DROP TABLE IF EXISTS usage; DROP TABLE IF EXISTS usage_sync;") + conn.executescript(SCHEMA_SQL) conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") conn.commit() + else: + conn.executescript(SCHEMA_SQL) return conn def refresh(self, conn: sqlite3.Connection) -> int: """Pull new rows from every session's usage.jsonl into the DB. - Returns the total number of rows inserted across all sessions. - Anomalies (shrunk sidecars, malformed lines) are logged but do not - raise. + Returns the total number of rows written (inserted or updated) across + all sessions. Anomalies (shrunk sidecars, malformed lines) are logged + but do not raise. """ - inserted = 0 + written = 0 root = sessions_root(self.thirdeye_home) if not root.exists(): return 0 @@ -63,9 +101,9 @@ def refresh(self, conn: sqlite3.Connection) -> int: for session_dir_ in sorted(platform_dir_.iterdir()): if not session_dir_.is_dir(): continue - inserted += self._refresh_one(conn, session_dir_.name, session_dir_) + written += self._refresh_one(conn, session_dir_.name, session_dir_) conn.commit() - return inserted + return written def refresh_session(self, conn: sqlite3.Connection, session_id: str, session_dir_: Path) -> int: n = self._refresh_one(conn, session_id, session_dir_) @@ -99,50 +137,43 @@ def _refresh_one(self, conn: sqlite3.Connection, sid: str, session_dir_: Path) - conn.execute("DELETE FROM usage WHERE session_id = ?", (sid,)) last_size = 0 - inserted = 0 - with jsonl.open("rb") as f: - f.seek(last_size) - for raw in f: - line = raw.decode("utf-8", errors="replace").strip() - if not line: - continue - try: - row = json.loads(line) - except json.JSONDecodeError: - log_capture_error( - thirdeye_home=self.thirdeye_home, - phase="index_sync", - message="malformed jsonl line", - session_id=sid, - source_path=str(jsonl), - ) - continue - try: - cursor = conn.execute( - "INSERT OR IGNORE INTO usage " - "(session_id, seq, ts, platform, model, " - "input_tokens, output_tokens, total_tokens) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - ( - row["session_id"], - int(row["seq"]), - row["ts"], - row["platform"], - row["model"], - int(row["input_tokens"]), - int(row["output_tokens"]), - int(row["total_tokens"]), - ), - ) - inserted += cursor.rowcount - except (KeyError, ValueError, sqlite3.Error) as e: - log_capture_error( - thirdeye_home=self.thirdeye_home, - phase="index_sync", - error=e, - session_id=sid, - source_path=str(jsonl), - ) + # The sidecar has grown. iter_calls is last-wins over the whole file, so + # re-read it entirely and upsert every logical call; a later append that + # corrects a call overwrites the earlier value. Malformed lines are + # silently dropped by iter_calls, so scan the grown region separately to + # log them. + self._log_malformed(sid, jsonl, last_size) + + written = 0 + for row in iter_calls(session_dir_): + try: + cursor = conn.execute( + _INSERT_SQL, + ( + row.session_id, + row.call_id, + row.seq, + row.ts, + row.platform, + row.provider_name, + row.operation_name, + row.response_model, + row.input_tokens, + row.output_tokens, + row.cache_read_input_tokens, + row.cache_creation_input_tokens, + row.reasoning_output_tokens, + ), + ) + written += cursor.rowcount + except sqlite3.Error as e: + log_capture_error( + thirdeye_home=self.thirdeye_home, + phase="index_sync", + error=e, + session_id=sid, + source_path=str(jsonl), + ) conn.execute( "INSERT INTO usage_sync (session_id, last_jsonl_size) " @@ -150,4 +181,31 @@ def _refresh_one(self, conn: sqlite3.Connection, sid: str, session_dir_: Path) - "ON CONFLICT (session_id) DO UPDATE SET last_jsonl_size = excluded.last_jsonl_size", (sid, current_size), ) - return inserted + return written + + def _log_malformed(self, sid: str, jsonl: Path, from_offset: int) -> None: + """Log any lines that are not valid JSON in the grown region. + + Purely observability: the actual upsert reads through iter_calls, which + drops malformed lines. Scans only from ``from_offset`` to avoid + re-logging lines seen in a prior refresh. + """ + try: + with jsonl.open("rb") as f: + f.seek(from_offset) + for raw in f: + line = raw.decode("utf-8", errors="replace").strip() + if not line: + continue + try: + json.loads(line) + except json.JSONDecodeError: + log_capture_error( + thirdeye_home=self.thirdeye_home, + phase="index_sync", + message="malformed jsonl line", + session_id=sid, + source_path=str(jsonl), + ) + except OSError: + return From be95785c990f91482f31afd08d4c5bc013496ec4 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Wed, 5 Aug 2026 23:49:02 -0700 Subject: [PATCH 10/15] Update usage web views for OTel GenAI schema Session usage now reads through usage.read.iter_calls so repeated per-frame rows collapse to one logical call. The session table renders the new per-call fields: response_model as a model column, plus cache read / cache creation / reasoning columns that show '-' for an absent (None) value and '0' for a reported zero. The global platform filter drops the removed gemini platform, leaving claude and codex. Co-Authored-By: Claude Opus 4.8 --- src/thirdeye/web/routes/usage.py | 4 +- src/thirdeye/web/templates/usage/global.html | 2 +- src/thirdeye/web/templates/usage/session.html | 8 +- tests/web/test_routes_usage.py | 126 ++++++++++++++++++ tests/web/test_usage_charts.py | 59 +++++++- 5 files changed, 189 insertions(+), 10 deletions(-) diff --git a/src/thirdeye/web/routes/usage.py b/src/thirdeye/web/routes/usage.py index 01bafe4..54483ff 100644 --- a/src/thirdeye/web/routes/usage.py +++ b/src/thirdeye/web/routes/usage.py @@ -12,7 +12,7 @@ from thirdeye.paths import session_dir from thirdeye.timeparse import parse_when from thirdeye.usage.aggregate import aggregate_by_day -from thirdeye.usage.store import UsageStore +from thirdeye.usage.read import iter_calls async def _session_usage(request: Request) -> HTMLResponse: @@ -24,7 +24,7 @@ async def _session_usage(request: Request) -> HTMLResponse: except (KeyError, ValueError) as e: raise HTTPException(status_code=404, detail=str(e)) from e sdir = session_dir(config.root, platform, sid) - rows = list(UsageStore(sdir).iter_rows()) + rows = list(iter_calls(sdir)) aggregate = store.stats(session_id=sid) templates = request.app.state.templates return templates.TemplateResponse( diff --git a/src/thirdeye/web/templates/usage/global.html b/src/thirdeye/web/templates/usage/global.html index c1a1905..db9f3f7 100644 --- a/src/thirdeye/web/templates/usage/global.html +++ b/src/thirdeye/web/templates/usage/global.html @@ -11,7 +11,7 @@