From f32b86c656a4e5f30d67bedacb06e7ef21a7908b Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Sun, 4 Oct 2026 13:02:48 +0300 Subject: [PATCH] Add a prompt-cache report and per-day cache diagnostics Add `statsai report cache`, which shows how much logical input the prompt cache served, how much context each model call processed, and when reuse dropped. For Claude Code and Codex it walks each agent's calls in order and flags a suspected cache loss when cached reads fall at least 50% and 4,096 tokens below what the previous call left reusable. Each loss records the reusable context it missed and its cost: the call's price minus its price had that context been read from the cache, priced as one request by the call's own model. Losses are grouped by the gap since the previous call. Model changes, compactions, and calls that cannot be ordered or split are boundaries, not losses. A Claude Code call that neither read nor wrote the cache, as when a custom endpoint serves it, is counted apart and not judged. The report takes --from/--to, --all, --provider, --account, --session, --timeline for local 10-minute buckets, --details for per-call evidence, and --json. The daemon serves the same report at /reports/cache, and `statsai schema cache-report` prints its schema. - Codex turn events list each model call with its request start, model, and compaction boundary. Rows too large to read still mark compaction. - Claude Code events record their request start, sub-agent, and compaction boundary, following file order because compaction rewrites history. - Daily summaries carry an optional, compact `cache_health` object. It is sent only to receivers that advertise support, and an existing store gains it once, rewriting only summaries whose content changes. Fields that are zero on most days are left out then. - The cache hit ratio now counts cache writes in its denominator. - Migration 30 orders the session index by start time, so sessions read in order and the calls after a changed one are an index range. --- Cargo.lock | 1 + README.md | 28 + crates/statsai-adapters/src/cache.rs | 8 +- crates/statsai-adapters/src/claude/parse.rs | 126 ++- .../src/claude/tests/parse.rs | 201 ++++ .../statsai-adapters/src/codex/parse/line.rs | 4 + .../statsai-adapters/src/codex/parse/mod.rs | 117 ++- .../statsai-adapters/src/codex/parse/types.rs | 50 + .../src/codex/tests/parse/calls.rs | 219 +++++ .../src/codex/tests/parse/mod.rs | 1 + crates/statsai-adapters/src/event.rs | 2 + crates/statsai-adapters/src/grok/parse/mod.rs | 1 + crates/statsai-adapters/src/json.rs | 19 +- crates/statsai-core/src/cache/detector.rs | 554 +++++++++++ crates/statsai-core/src/cache/health.rs | 246 +++++ crates/statsai-core/src/cache/mod.rs | 97 ++ crates/statsai-core/src/cache/report.rs | 346 +++++++ crates/statsai-core/src/lib.rs | 2 + crates/statsai-core/src/tests/cache.rs | 849 +++++++++++++++++ crates/statsai-core/src/tests/mod.rs | 1 + crates/statsai-core/src/tests/support.rs | 1 + crates/statsai-core/src/types/event.rs | 52 ++ crates/statsai-core/src/types/summary.rs | 43 + crates/statsai-daemon/Cargo.toml | 1 + crates/statsai-daemon/src/lib.rs | 51 ++ crates/statsai-daemon/src/tests.rs | 124 +++ crates/statsai-sdk/src/lib.rs | 5 + .../src/account_plan/observations.rs | 3 +- crates/statsai-store/src/cache.rs | 614 +++++++++++++ crates/statsai-store/src/events.rs | 22 +- crates/statsai-store/src/lib.rs | 19 +- crates/statsai-store/src/migrations/mod.rs | 42 +- crates/statsai-store/src/migrations/v2.rs | 19 + .../src/pricing/tests/repricing.rs | 2 +- .../src/pricing/tests/support.rs | 1 + .../statsai-store/src/quota/tests/support.rs | 1 + crates/statsai-store/src/rollups/build.rs | 38 +- crates/statsai-store/src/rollups/mod.rs | 29 +- crates/statsai-store/src/sync_state/mod.rs | 7 +- .../statsai-store/src/sync_state/pending.rs | 23 +- crates/statsai-store/src/tests/cache.rs | 636 +++++++++++++ crates/statsai-store/src/tests/events.rs | 1 + crates/statsai-store/src/tests/mod.rs | 1 + .../statsai-store/src/tests/rollups/build.rs | 4 +- crates/statsai-store/src/tests/support.rs | 1 + .../src/tests/sync_state/pending.rs | 3 + .../tests/cache_health_fixture.rs | 304 ++++++ .../tests/fixtures/cache_health.json | 785 ++++++++++++++++ crates/statsai-sync/src/lib.rs | 1 + crates/statsai/src/cli/args/mod.rs | 2 + crates/statsai/src/cli/args/report.rs | 27 + crates/statsai/src/cli/cache.rs | 363 ++++++++ crates/statsai/src/cli/mod.rs | 1 + crates/statsai/src/cli/report.rs | 4 + crates/statsai/src/cli/schema.rs | 6 +- crates/statsai/src/cli/sync/batch.rs | 24 +- crates/statsai/src/cli/sync/http.rs | 9 + crates/statsai/src/cli/sync/mod.rs | 16 +- crates/statsai/src/cli/tests/args.rs | 24 + crates/statsai/src/cli/tests/support.rs | 1 + crates/statsai/src/cli/tests/sync/batch.rs | 62 ++ crates/statsai/src/cli/tests/sync/http.rs | 8 +- crates/statsai/src/cli/tests/sync/mod.rs | 6 + crates/statsai/src/lib.rs | 1 + crates/statsai/tests/cli_surface.rs | 7 + crates/statsai/tests/cli_surface/help.txt | 29 + .../cli_surface/schema-cache-report.json | 863 ++++++++++++++++++ .../tests/cli_surface/schema-sync-batch.json | 153 ++++ crates/statsai/tests/report_cli.rs | 1 + 69 files changed, 7230 insertions(+), 82 deletions(-) create mode 100644 crates/statsai-adapters/src/codex/tests/parse/calls.rs create mode 100644 crates/statsai-core/src/cache/detector.rs create mode 100644 crates/statsai-core/src/cache/health.rs create mode 100644 crates/statsai-core/src/cache/mod.rs create mode 100644 crates/statsai-core/src/cache/report.rs create mode 100644 crates/statsai-core/src/tests/cache.rs create mode 100644 crates/statsai-store/src/cache.rs create mode 100644 crates/statsai-store/src/tests/cache.rs create mode 100644 crates/statsai-store/tests/cache_health_fixture.rs create mode 100644 crates/statsai-store/tests/fixtures/cache_health.json create mode 100644 crates/statsai/src/cli/cache.rs create mode 100644 crates/statsai/tests/cli_surface/schema-cache-report.json diff --git a/Cargo.lock b/Cargo.lock index 7fc408a..d676a2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2929,6 +2929,7 @@ dependencies = [ "libc", "log", "notify", + "percent-encoding", "serde", "serde_json", "statsai-adapters", diff --git a/README.md b/README.md index 1670ac0..af87a47 100644 --- a/README.md +++ b/README.md @@ -323,6 +323,28 @@ The benchmark becomes a meaningful quality gate after you record verified ground truth. See [`docs/task-collection.md`](docs/task-collection.md) and [`docs/task-benchmarking.md`](docs/task-benchmarking.md). +### See how well the prompt cache is working + +```sh +statsai report cache +statsai report cache --from 2026-06-01 --to 2026-06-07 --provider claude_code +statsai report cache --all --provider codex +statsai report cache --session session_1a2b --details +statsai report cache --timeline --json +``` + +The cache report shows how much logical input (ordinary input, cache writes, +and cache reads) was served from the prompt cache, how much context each model +call processed, and when reuse dropped. For Claude Code and Codex it also walks +each agent's calls in order: a call whose cached reads fall at least half, and +at least 4,096 tokens, below what the previous call left reusable is a +suspected cache loss. Losses are grouped by the gap since the previous call, so +drops after a pause read apart from drops during active use. It is a heuristic +over recorded token counts, not a provider-confirmed miss, and calls that +cannot be ordered or split are reported as unclassifiable rather than as +healthy. `--timeline` adds local 10-minute buckets and `--details` lists every +call with its evidence; per-call details never leave the device. + ### Keep a durable conversation archive Collect complete conversations into the local SQLite store, then search or @@ -394,6 +416,7 @@ statsai report monthly --subscriptions statsai report all-time --json --verbose statsai report range --from 2026-01-01 --to 2026-03-31 statsai report range --from 2026-05-01 --json +statsai report cache --from 2026-05-01 --details ``` Normal scans use a per-source file signature cache. `--no-cache` forces a @@ -439,6 +462,7 @@ statsai sync --sink http --since-last statsai sync --sink http --verify statsai sync --status statsai schema sync-batch +statsai schema cache-report ``` HTTP sync uses the stored device session unless `--auth-token` or @@ -459,6 +483,10 @@ curl -H "Authorization: Bearer $(cat ~/.statsai/daemon-token)" \ http://127.0.0.1:8765/accounts ``` +`/reports/cache` returns the `statsai report cache --json` report and accepts +`from`, `to`, `all`, `provider`, `account`, `session`, and `timeline` query +parameters. + Browser-originated requests are rejected. Sync writes must use `Content-Type: application/json` and stay below 8 MiB. diff --git a/crates/statsai-adapters/src/cache.rs b/crates/statsai-adapters/src/cache.rs index e3b9edd..3666c18 100644 --- a/crates/statsai-adapters/src/cache.rs +++ b/crates/statsai-adapters/src/cache.rs @@ -15,11 +15,15 @@ pub(crate) const SCAN_CACHE_SIGNATURE_VERSION: &str = "scan-cache.v1"; // parent's name, and turns without a reported duration end at their last work. // session-metadata.v36: sub-agents report the parent they borrow a name from, // so the name is joined when the session is built. -pub(crate) const CODEX_SCAN_CACHE_PARSER_REVISION: &str = "session-metadata.v36"; +// call-context.v38: turn events list their model calls, with request starts, +// compaction boundaries, and each call's model, for the prompt-cache report. +pub(crate) const CODEX_SCAN_CACHE_PARSER_REVISION: &str = "call-context.v38"; // session-metadata.v30: Claude user-message counts include only typed prompts, // not tool results, meta lines, or compaction summaries. (v29: events carry the // session's custom or AI title and the prompt that started their turn.) -pub(crate) const CLAUDE_SCAN_CACHE_PARSER_REVISION: &str = "session-metadata.v30"; +// call-context.v31: events record their request start, sub-agent, and +// compaction boundary for the prompt-cache report. +pub(crate) const CLAUDE_SCAN_CACHE_PARSER_REVISION: &str = "call-context.v31"; // session-metadata.v21: events carry each session's user and assistant message // counts. (v20: session rows report their message count as requests and are // priced per message, so long-context tiers stop being decided session-wide.) diff --git a/crates/statsai-adapters/src/claude/parse.rs b/crates/statsai-adapters/src/claude/parse.rs index d27275a..fb87194 100644 --- a/crates/statsai-adapters/src/claude/parse.rs +++ b/crates/statsai-adapters/src/claude/parse.rs @@ -50,6 +50,7 @@ pub(crate) fn parse_claude_file( // counted on the first snapshot have to ride along with that replacement. let mut last_usage_users = HashMap::::new(); let mut turn_started_at = HashMap::>::new(); + let mut call_streams = HashMap::<(String, Option), ClaudeCallStream>::new(); loop { let line_status = read_bounded_jsonl_line(&mut reader, &mut line_bytes, MAX_JSONL_RECORD_BYTES)?; @@ -84,6 +85,11 @@ pub(crate) fn parse_claude_file( current_reasoning = reasoning; } let session_raw = claude_session_raw(&value, path); + let agent_hash = claude_agent_hash(&value); + call_streams + .entry((session_raw.clone(), agent_hash.clone())) + .or_default() + .observe(&value); if let Some((rank, title)) = claude_session_title_from_value(&value) { session_titles.remember(session_raw, rank, title); continue; @@ -176,7 +182,7 @@ pub(crate) fn parse_claude_file( model, usage, runtime: message_count_runtime(user_messages, 1, 0), - session_raw, + session_raw: session_raw.clone(), project, event_kind: "claude_message_usage", source_file: path, @@ -189,6 +195,9 @@ pub(crate) fn parse_claude_file( }, ); event.session.turn_started_at = prompted_at.filter(|prompted_at| *prompted_at <= timestamp); + event.context = call_streams + .get_mut(&(session_raw, agent_hash.clone())) + .and_then(|stream| stream.call_context(record_id.as_deref(), timestamp, agent_hash)); push_deduped(ctx.scan, ctx.seen, event, duplicate_selection); } @@ -204,6 +213,121 @@ pub(crate) fn parse_claude_file( Ok(()) } +/// Prompt-cache context for one agent's calls in one session. +/// +/// A request goes out when Claude Code records the prompt or tool result that +/// triggers it, so the latest such record before a response is the request +/// start. A streamed response is written as several records sharing one +/// provider-record id; all of them describe the same call, and so does any +/// later copy of it, which keeps the marks its first record got. +/// +/// Transcripts are not in time order: compaction rewrites the messages it +/// keeps after its boundary, with their original timestamps. Everything here +/// therefore follows file order, and a record older than the compaction +/// boundary is one of those copies, not the call that follows it. +#[derive(Default)] +struct ClaudeCallStream { + last_input_at: Option>, + last_completed_at: Option>, + pending_compaction: Option, + marks_by_record: HashMap, +} + +#[derive(Clone, Copy)] +enum ClaudeCompaction { + At(chrono::DateTime), + Unplaced, +} + +#[derive(Clone, Copy)] +struct ClaudeCallMarks { + requested_at: Option>, + after_compaction: bool, +} + +impl ClaudeCallStream { + fn observe(&mut self, value: &Value) { + if claude_record_is_compact_boundary(value) { + self.pending_compaction = Some( + timestamp_from_nested_value(value) + .map_or(ClaudeCompaction::Unplaced, ClaudeCompaction::At), + ); + } else if claude_record_is_user(value) { + if let Some(timestamp) = timestamp_from_nested_value(value) { + self.last_input_at = Some(timestamp); + } + } + } + + fn call_context( + &mut self, + record_id: Option<&str>, + completed_at: chrono::DateTime, + agent_hash: Option, + ) -> Option { + let known = record_id.and_then(|record_id| self.marks_by_record.get(record_id)); + let marks = match known { + Some(marks) => { + // A later record of a known call: another streamed part, or a + // copy compaction wrote with the original timestamp. + let marks = *marks; + self.last_completed_at = self.last_completed_at.max(Some(completed_at)); + marks + } + None => { + // Input recorded before the previous response belongs to an + // earlier exchange, as when a call retries without new input. + let previous = self.last_completed_at; + let after_compaction = match self.pending_compaction { + Some(ClaudeCompaction::At(at)) if completed_at < at => false, + Some(_) => { + self.pending_compaction = None; + true + } + None => false, + }; + let marks = ClaudeCallMarks { + requested_at: self.last_input_at.filter(|at| { + *at <= completed_at && previous.is_none_or(|previous| *at >= previous) + }), + after_compaction, + }; + if let Some(record_id) = record_id { + self.marks_by_record.insert(record_id.to_string(), marks); + } + self.last_completed_at = Some(completed_at); + marks + } + }; + let context = statsai_core::CallContext { + agent_hash, + requested_at: marks.requested_at, + after_compaction: marks.after_compaction, + calls: Vec::new(), + }; + (context != statsai_core::CallContext::default()).then_some(context) + } +} + +/// Sub-agent identity. Sub-agents write their own transcripts, marked as +/// sidechains, under the parent's session id. +fn claude_agent_hash(value: &Value) -> Option { + if let Some(agent_id) = value + .get("agentId") + .and_then(Value::as_str) + .filter(|agent_id| !agent_id.trim().is_empty()) + { + return Some(hash_text(&format!("claude_agent:{agent_id}"))[..24].to_string()); + } + (value.get("isSidechain").and_then(Value::as_bool) == Some(true)) + .then(|| hash_text("claude_agent:sidechain")[..24].to_string()) +} + +fn claude_record_is_compact_boundary(value: &Value) -> bool { + value.get("type").and_then(Value::as_str) == Some("system") + && value.get("subtype").and_then(Value::as_str) == Some("compact_boundary") +} + /// Session names found across one source's transcripts. /// /// A resumed or forked session copies earlier messages into a new transcript, diff --git a/crates/statsai-adapters/src/claude/tests/parse.rs b/crates/statsai-adapters/src/claude/tests/parse.rs index 9cf184e..5570abf 100644 --- a/crates/statsai-adapters/src/claude/tests/parse.rs +++ b/crates/statsai-adapters/src/claude/tests/parse.rs @@ -1373,3 +1373,204 @@ fn claude_title_only_transcripts_report_session_names() { }] ); } + +#[test] +fn claude_events_record_request_starts_sub_agents_and_compaction() { + let dir = tempfile::tempdir().expect("tempdir"); + let project = dir.path().join("projects").join("example"); + let agents = project.join("session-1").join("subagents"); + std::fs::create_dir_all(&agents).expect("projects"); + let assistant = |timestamp: &str, message: &str, request: &str, extra: &str| { + format!( + r#"{{"timestamp":"{timestamp}","sessionId":"session-1"{extra},"type":"assistant","requestId":"{request}","message":{{"id":"{message}","role":"assistant","model":"claude-opus-5","usage":{{"input_tokens":2,"cache_creation_input_tokens":100,"cache_read_input_tokens":9000,"output_tokens":5}}}}}}"# + ) + }; + let main = [ + r#"{"timestamp":"2026-08-05T10:00:00Z","sessionId":"session-1","type":"user","message":{"role":"user","content":"hello"}}"#.to_string(), + assistant("2026-08-05T10:00:05Z", "m1", "r1", ""), + assistant("2026-08-05T10:00:07Z", "m1", "r1", ""), + r#"{"timestamp":"2026-08-05T10:00:20Z","sessionId":"session-1","type":"user","toolUseResult":{},"message":{"role":"user","content":[{"type":"tool_result","content":"ok"}]}}"#.to_string(), + r#"{"timestamp":"2026-08-05T10:00:30Z","sessionId":"session-1","type":"system","subtype":"compact_boundary"}"#.to_string(), + r#"{"timestamp":"2026-08-05T10:00:31Z","sessionId":"session-1","type":"user","isCompactSummary":true,"message":{"role":"user","content":"summary"}}"#.to_string(), + assistant("2026-08-05T10:00:40Z", "m2", "r2", ""), + assistant("2026-08-05T10:00:50Z", "m4", "r4", ""), + ]; + std::fs::write(project.join("session-1.jsonl"), main.join("\n") + "\n").expect("main"); + let agent = [ + r#"{"timestamp":"2026-08-05T10:00:08Z","sessionId":"session-1","agentId":"a1","isSidechain":true,"type":"user","message":{"role":"user","content":"task"}}"#.to_string(), + assistant( + "2026-08-05T10:00:12Z", + "m3", + "r3", + r#","agentId":"a1","isSidechain":true"#, + ), + ]; + std::fs::write(agents.join("agent-a1.jsonl"), agent.join("\n") + "\n").expect("agent"); + + let source = SourceLocation::local_adapter( + CLAUDE_CODE_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let scan = scan_claude_source(&ClaudeCodeAdapter, &source, &options()).expect("scan"); + let event_at = |timestamp: &str| { + let at = timestamp + .parse::>() + .expect("timestamp"); + scan.events + .iter() + .find(|event| event.created_at == at) + .unwrap_or_else(|| panic!("event at {timestamp}")) + }; + let at = |timestamp: &str| Some(timestamp.parse::>().expect("time")); + + assert_eq!(scan.events.len(), 4, "streamed snapshots are one call"); + let first = event_at("2026-08-05T10:00:07Z") + .context + .as_ref() + .expect("context"); + assert_eq!(first.requested_at, at("2026-08-05T10:00:00Z")); + assert!(!first.after_compaction); + assert_eq!(first.agent_hash, None); + + let compacted = event_at("2026-08-05T10:00:40Z") + .context + .as_ref() + .expect("context"); + assert_eq!(compacted.requested_at, at("2026-08-05T10:00:31Z")); + assert!(compacted.after_compaction); + + // No input since the previous response: no request start, and the + // compaction marked only the call after it. + assert_eq!(event_at("2026-08-05T10:00:50Z").context, None); + + let sub_agent = event_at("2026-08-05T10:00:12Z") + .context + .as_ref() + .expect("context"); + assert!(sub_agent.agent_hash.is_some()); + assert_eq!(sub_agent.requested_at, at("2026-08-05T10:00:08Z")); +} + +#[test] +fn claude_compaction_copies_do_not_shift_request_starts_or_the_boundary() { + let dir = tempfile::tempdir().expect("tempdir"); + let project = dir.path().join("projects").join("example"); + std::fs::create_dir_all(&project).expect("projects"); + let user = |timestamp: &str| { + format!( + r#"{{"timestamp":"{timestamp}","sessionId":"session-1","type":"user","message":{{"role":"user","content":"go"}}}}"# + ) + }; + let assistant = |timestamp: &str, message: &str| { + format!( + r#"{{"timestamp":"{timestamp}","sessionId":"session-1","type":"assistant","requestId":"r-{message}","message":{{"id":"{message}","role":"assistant","model":"claude-opus-5","usage":{{"input_tokens":2,"cache_read_input_tokens":9000,"output_tokens":5}}}}}}"# + ) + }; + let lines = [ + user("2026-08-05T10:00:00Z"), + assistant("2026-08-05T10:00:05Z", "m1"), + user("2026-08-05T11:00:00Z"), + assistant("2026-08-05T11:00:05Z", "m2"), + r#"{"timestamp":"2026-08-05T11:30:00Z","sessionId":"session-1","type":"system","subtype":"compact_boundary"}"#.to_string(), + // The preserved segment, rewritten with its original timestamps. + user("2026-08-05T10:00:00Z"), + assistant("2026-08-05T10:00:05Z", "m1"), + user("2026-08-05T11:31:00Z"), + assistant("2026-08-05T11:31:05Z", "m3"), + ]; + std::fs::write(project.join("session-1.jsonl"), lines.join("\n") + "\n").expect("transcript"); + let source = SourceLocation::local_adapter( + CLAUDE_CODE_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let scan = scan_claude_source(&ClaudeCodeAdapter, &source, &options()).expect("scan"); + let context = |timestamp: &str| { + let at = timestamp + .parse::>() + .expect("timestamp"); + scan.events + .iter() + .find(|event| event.created_at == at) + .and_then(|event| event.context.clone()) + .unwrap_or_default() + }; + + assert_eq!(scan.events.len(), 3); + assert!( + !context("2026-08-05T10:00:05Z").after_compaction, + "a copy is not the next call" + ); + let next = context("2026-08-05T11:31:05Z"); + assert!(next.after_compaction); + assert_eq!( + next.requested_at, + Some("2026-08-05T11:31:00Z".parse().expect("timestamp")) + ); +} + +#[test] +fn claude_compaction_copies_keep_the_marks_of_the_call_they_copy() { + let dir = tempfile::tempdir().expect("tempdir"); + let project = dir.path().join("projects").join("example"); + std::fs::create_dir_all(&project).expect("projects"); + let user = |timestamp: &str| { + format!( + r#"{{"timestamp":"{timestamp}","sessionId":"session-1","type":"user","message":{{"role":"user","content":"go"}}}}"# + ) + }; + let assistant = |timestamp: &str, message: &str| { + format!( + r#"{{"timestamp":"{timestamp}","sessionId":"session-1","type":"assistant","requestId":"r-{message}","message":{{"id":"{message}","role":"assistant","model":"claude-opus-5","usage":{{"input_tokens":2,"cache_read_input_tokens":9000,"output_tokens":5}}}}}}"# + ) + }; + let boundary = |timestamp: &str| { + format!( + r#"{{"timestamp":"{timestamp}","sessionId":"session-1","type":"system","subtype":"compact_boundary"}}"# + ) + }; + let lines = [ + user("2026-08-05T10:00:00Z"), + assistant("2026-08-05T10:00:05Z", "m1"), + boundary("2026-08-05T10:30:00Z"), + user("2026-08-05T10:31:00Z"), + assistant("2026-08-05T10:31:05Z", "m2"), + boundary("2026-08-05T11:30:00Z"), + // A second compaction keeps the call that followed the first one. + user("2026-08-05T10:31:00Z"), + assistant("2026-08-05T10:31:05Z", "m2"), + ]; + std::fs::write(project.join("session-1.jsonl"), lines.join("\n") + "\n").expect("transcript"); + let source = SourceLocation::local_adapter( + CLAUDE_CODE_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let scan = scan_claude_source(&ClaudeCodeAdapter, &source, &options()).expect("scan"); + let at = "2026-08-05T10:31:05Z" + .parse::>() + .expect("timestamp"); + let copied = scan + .events + .iter() + .filter(|event| event.created_at == at) + .collect::>(); + + assert_eq!(copied.len(), 1); + let context = copied[0].context.clone().unwrap_or_default(); + assert!( + context.after_compaction, + "the copy keeps its compaction mark" + ); + assert_eq!( + context.requested_at, + Some("2026-08-05T10:31:00Z".parse().expect("timestamp")) + ); +} diff --git a/crates/statsai-adapters/src/codex/parse/line.rs b/crates/statsai-adapters/src/codex/parse/line.rs index c19a3ff..9e63d59 100644 --- a/crates/statsai-adapters/src/codex/parse/line.rs +++ b/crates/statsai-adapters/src/codex/parse/line.rs @@ -48,6 +48,8 @@ pub(crate) enum CodexLineKind { TokenCount, TokenUsageRecord, Compacted, + /// A remote compaction's replacement history, written as a response item. + ResponseItemCompaction, TaskStarted, TaskComplete, HeadlessUsage, @@ -114,6 +116,8 @@ pub(crate) fn codex_line_kind(line: &str) -> CodexLineKind { || header.contains("\"payload\":{\"type\":\"tool_search_call\"") { CodexLineKind::ResponseItemToolCall + } else if header.contains("\"payload\":{\"type\":\"compaction\"") { + CodexLineKind::ResponseItemCompaction } else { CodexLineKind::Irrelevant }; diff --git a/crates/statsai-adapters/src/codex/parse/mod.rs b/crates/statsai-adapters/src/codex/parse/mod.rs index 054d0fa..1ea485f 100644 --- a/crates/statsai-adapters/src/codex/parse/mod.rs +++ b/crates/statsai-adapters/src/codex/parse/mod.rs @@ -54,6 +54,7 @@ pub(crate) fn parse_codex_file( let mut line_bytes = Vec::new(); let mut index = 0usize; let mut activity = crate::activity::CodexActivityExtractor::default(); + let mut call_clock = CodexCallClock::default(); loop { let line_status = @@ -65,6 +66,18 @@ pub(crate) fn parse_codex_file( if line_status == BoundedLineRead::Oversized { ctx.scan.diagnostics.raw_rows += 1; ctx.scan.diagnostics.oversized_rows += 1; + // A compaction row carries the whole replacement history and can + // outgrow the record limit. Its header still marks the boundary. + let header = String::from_utf8_lossy(&line_bytes); + if matches!( + codex_line_kind(&header), + CodexLineKind::Compacted | CodexLineKind::ResponseItemCompaction + ) { + let session = + codex_json_string_prefix_after_marker(&header, "\"session_id\":\"", 256) + .unwrap_or_else(|| session_raw.clone()); + call_clock.compaction(&session); + } continue; } let Ok(line) = std::str::from_utf8(&line_bytes) else { @@ -95,6 +108,18 @@ pub(crate) fn parse_codex_file( continue; } if line_kind == CodexLineKind::ResponseItemToolCall { + let header = codex_line_header(line); + if header.contains("\"payload\":{\"type\":\"function_call_output\"") + || header.contains("\"payload\":{\"type\":\"custom_tool_call_output\"") + { + if let (timestamp, false) = codex_timestamp_from_text( + codex_json_string_prefix_after_marker(header, "\"timestamp\":\"", 64) + .as_deref(), + fallback_timestamp, + ) { + call_clock.input(&session_raw, timestamp); + } + } let activity_started_at = std::time::Instant::now(); activity.observe_legacy_line( ctx.source, @@ -118,6 +143,13 @@ pub(crate) fn parse_codex_file( ) .unwrap_or_else(|| session_raw.clone()); unpaired_records.remove(&session); + call_clock.compaction(&session); + continue; + } + if line_kind == CodexLineKind::ResponseItemCompaction { + // Pairing is left alone: unlike `compacted`, this row does not end + // a token_usage_record's pairing window. + call_clock.compaction(&session_raw); continue; } if line_kind == CodexLineKind::Irrelevant && !is_codex_quota_line_structurally(line) { @@ -148,6 +180,8 @@ pub(crate) fn parse_codex_file( ); if timestamp_inferred { ctx.scan.diagnostics.timestamp_fallbacks += 1; + } else if codex_role_sends_input(role.as_deref()) { + call_clock.input(&session_raw, timestamp); } let mut model_inferred = false; let model = current_model @@ -197,6 +231,8 @@ pub(crate) fn parse_codex_file( task_completed_at: None, task_duration_ms: None, time_to_first_token_ms: None, + requested_at: None, + after_compaction: false, }); continue; } @@ -208,6 +244,8 @@ pub(crate) fn parse_codex_file( codex_timestamp_from_text(parsed.timestamp.as_deref(), fallback_timestamp); if timestamp_inferred { ctx.scan.diagnostics.timestamp_fallbacks += 1; + } else if codex_role_sends_input(parsed.payload.role.as_deref()) { + call_clock.input(&session_raw, timestamp); } let mut model_inferred = false; let model = current_model @@ -263,18 +301,23 @@ pub(crate) fn parse_codex_file( task_completed_at: None, task_duration_ms: None, time_to_first_token_ms: None, + requested_at: None, + after_compaction: false, }); continue; } if line_kind == CodexLineKind::EventUserMessage { - if !collect_tasks { - continue; - } let header = codex_line_header(line); let (timestamp, timestamp_inferred) = codex_timestamp_from_text( codex_json_string_prefix_after_marker(header, "\"timestamp\":\"", 64).as_deref(), fallback_timestamp, ); + if !timestamp_inferred { + call_clock.input(&session_raw, timestamp); + } + if !collect_tasks { + continue; + } if timestamp_inferred { ctx.scan.diagnostics.timestamp_fallbacks += 1; } @@ -326,6 +369,8 @@ pub(crate) fn parse_codex_file( task_completed_at: None, task_duration_ms: None, time_to_first_token_ms: None, + requested_at: None, + after_compaction: false, }); continue; } @@ -456,6 +501,11 @@ pub(crate) fn parse_codex_file( .unwrap_or((fallback_timestamp, true)); if timestamp_inferred { ctx.scan.diagnostics.timestamp_fallbacks += 1; + } else if is_task_started { + call_clock.input( + &event_session_raw, + task_started_at.unwrap_or(timestamp).min(timestamp), + ); } if let Some(quota) = codex_quota_observation( ctx.source, @@ -514,6 +564,12 @@ pub(crate) fn parse_codex_file( } }); + let (requested_at, after_compaction) = match usage.as_ref() { + Some(_) if !timestamp_inferred => { + call_clock.call(&event_session_raw, timestamp, is_token_usage_record) + } + _ => (None, false), + }; records.push(CodexLineRecord { line_number: index, timestamp, @@ -538,6 +594,8 @@ pub(crate) fn parse_codex_file( task_completed_at, task_duration_ms, time_to_first_token_ms, + requested_at, + after_compaction, }); } @@ -568,6 +626,12 @@ pub(crate) fn parse_codex_file( .unwrap_or_default(), quota_lines: Vec::new(), project: record.project.clone(), + calls: record + .usage + .as_ref() + .map(|usage| codex_model_call(record, usage)) + .into_iter() + .collect(), }); if record.usage.is_some() { consumed_usage_lines.insert(record.line_number); @@ -659,6 +723,7 @@ pub(crate) fn parse_codex_file( .map(|accumulated| sum_usage_counts(accumulated, &usage)) .unwrap_or_else(|| usage.clone()), ); + turn.calls.push(codex_model_call(record, &usage)); turn.last_usage = Some(usage); turn.usage_lines.push(record.line_number); } @@ -745,6 +810,24 @@ pub(crate) fn parse_codex_file( }, ); event.session.title = turn.title.clone(); + // The completion's own usage, when present, replaces the calls', + // but a compaction anywhere in the turn still came before it. + if record.usage.is_none() && !turn.calls.is_empty() { + event.context = Some(CallContext { + calls: turn.calls.clone(), + ..CallContext::default() + }); + } else if record.usage.is_some() { + let after_compaction = + record.after_compaction || turn.calls.iter().any(|call| call.after_compaction); + if record.requested_at.is_some() || after_compaction { + event.context = Some(CallContext { + requested_at: record.requested_at, + after_compaction, + ..CallContext::default() + }); + } + } let mut linked_quota_lines = turn.usage_lines.clone(); linked_quota_lines.extend_from_slice(&turn.quota_lines); if record.usage.is_some() { @@ -979,6 +1062,13 @@ pub(crate) fn parse_codex_file( }, ); event.session.title = session_title; + if record.requested_at.is_some() || record.after_compaction { + event.context = Some(CallContext { + requested_at: record.requested_at, + after_compaction: record.after_compaction, + ..CallContext::default() + }); + } // A record's paired token_count carries the quota sample for it. let mut linked_quota_lines = vec![record.line_number]; linked_quota_lines.extend(paired_quota_lines.get(&record.line_number)); @@ -1003,3 +1093,24 @@ pub(crate) fn parse_codex_file( Ok(()) } + +/// Prompts and developer instructions are input to the next request. +fn codex_role_sends_input(role: Option<&str>) -> bool { + matches!(role, Some("user" | "developer")) +} + +fn codex_model_call(record: &CodexLineRecord, usage: &UsageCounts) -> ModelCall { + ModelCall { + completed_at: record.timestamp, + requested_at: record.requested_at, + input_tokens: usage.input_tokens, + cache_read_tokens: usage.cache_read_tokens, + cache_creation_tokens: usage.cache_creation_tokens, + after_compaction: record.after_compaction, + model: record + .model + .as_ref() + .filter(|_| !record.model_inferred) + .and_then(statsai_core::cache_model_name), + } +} diff --git a/crates/statsai-adapters/src/codex/parse/types.rs b/crates/statsai-adapters/src/codex/parse/types.rs index ba0d1aa..a88368c 100644 --- a/crates/statsai-adapters/src/codex/parse/types.rs +++ b/crates/statsai-adapters/src/codex/parse/types.rs @@ -23,6 +23,10 @@ pub(crate) struct CodexLineRecord { pub(crate) task_completed_at: Option>, pub(crate) task_duration_ms: Option, pub(crate) time_to_first_token_ms: Option, + /// For usage lines: when the request was sent, if the rollout says. + pub(crate) requested_at: Option>, + /// For usage lines: the conversation was compacted before this call. + pub(crate) after_compaction: bool, } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -117,4 +121,50 @@ pub(crate) struct ActiveCodexTurn { /// observations still belong to this turn's event. pub(crate) quota_lines: Vec, pub(crate) project: Option, + /// The model calls whose usage the turn accumulates, in order. + pub(crate) calls: Vec, +} + +/// When each session's model calls were requested, for the prompt-cache +/// report. +/// +/// A `token_usage_record` is written as the response completes, before any +/// tool runs, so the latest prompt, tool output, or turn start after it is +/// when the next request went out. A `token_count` alone is written after the +/// response's tools have run, so it cannot place a request start. Rollouts can +/// carry copied history with older timestamps, so this follows file order. +#[derive(Default)] +pub(crate) struct CodexCallClock { + last_input_at: HashMap>, + last_call_at: HashMap>, + pending_compaction: HashSet, +} + +impl CodexCallClock { + pub(crate) fn input(&mut self, session_raw: &str, at: DateTime) { + self.last_input_at.insert(session_raw.to_string(), at); + } + + pub(crate) fn compaction(&mut self, session_raw: &str) { + self.pending_compaction.insert(session_raw.to_string()); + } + + /// Records a call completing and returns its request start, when the + /// rollout places one, and whether it follows a compaction. + pub(crate) fn call( + &mut self, + session_raw: &str, + completed_at: DateTime, + record_placed: bool, + ) -> (Option>, bool) { + let previous_call_at = self + .last_call_at + .insert(session_raw.to_string(), completed_at); + let requested_at = record_placed + .then(|| self.last_input_at.get(session_raw).copied()) + .flatten() + .filter(|at| *at <= completed_at) + .filter(|at| previous_call_at.is_none_or(|previous| *at >= previous)); + (requested_at, self.pending_compaction.remove(session_raw)) + } } diff --git a/crates/statsai-adapters/src/codex/tests/parse/calls.rs b/crates/statsai-adapters/src/codex/tests/parse/calls.rs new file mode 100644 index 0000000..a24300c --- /dev/null +++ b/crates/statsai-adapters/src/codex/tests/parse/calls.rs @@ -0,0 +1,219 @@ +use super::*; + +/// Codex reports cached input inside `input_tokens`. +fn usage_record(timestamp: &str, input: u64, cached: u64) -> String { + format!( + r#"{{"timestamp":"{timestamp}","type":"token_usage_record","payload":{{"usage":{{"input_tokens":{input},"cached_input_tokens":{cached},"cache_write_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0,"total_tokens":{}}}}}}}"#, + input + 50 + ) +} + +fn token_count(timestamp: &str, total: u64, input: u64, cached: u64) -> String { + format!( + r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"token_count","info":{{"total_token_usage":{{"input_tokens":{total},"cached_input_tokens":0,"output_tokens":0,"total_tokens":{total}}},"last_token_usage":{{"input_tokens":{input},"cached_input_tokens":{cached},"cache_write_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0,"total_tokens":{}}}}}}}}}"#, + input + 50 + ) +} + +fn scan_lines(lines: &[String]) -> AdapterScan { + let dir = tempfile::tempdir().expect("tempdir"); + let sessions = dir.path().join("sessions"); + std::fs::create_dir_all(&sessions).expect("sessions"); + std::fs::write(sessions.join("rollout.jsonl"), lines.join("\n") + "\n").expect("fixture"); + let source = SourceLocation::local_adapter( + CODEX_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + scan_codex_source(&CodexAdapter, &source, &options_without_tasks()).expect("scan") +} + +fn at(timestamp: &str) -> Option> { + Some(timestamp.parse().expect("timestamp")) +} + +fn turn_lines(compaction: String) -> Vec { + vec![ + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"session_meta","payload":{"id":"thread-1"}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"event_msg","payload":{"type":"task_started","turn_id":"turn-1"}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:01Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"go"}]}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:09Z","type":"response_item","payload":{"type":"custom_tool_call","name":"apply_patch","input":""}}"#.to_string(), + usage_record("2026-09-01T10:00:10Z", 30_000, 0), + r#"{"timestamp":"2026-09-01T10:00:12Z","type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"c1","output":"ok"}}"#.to_string(), + token_count("2026-09-01T10:00:12.500Z", 30_050, 30_000, 0), + usage_record("2026-09-01T10:00:20Z", 32_000, 29_952), + compaction, + usage_record("2026-09-01T10:00:30Z", 8_024, 1_024), + r#"{"timestamp":"2026-09-01T10:00:40Z","type":"event_msg","payload":{"type":"task_complete","turn_id":"turn-1"}}"#.to_string(), + ] +} + +#[test] +fn codex_turns_list_their_calls_with_request_starts_and_compaction() { + let scan = scan_lines(&turn_lines( + r#"{"timestamp":"2026-09-01T10:00:21Z","type":"compacted","payload":{"message":"","replacement_history":[]}}"#.to_string(), + )); + + assert_eq!(scan.events.len(), 1); + let event = &scan.events[0]; + assert_eq!( + event.usage.requests, + Some(3), + "the paired token_count is not a call" + ); + let calls = &event.context.as_ref().expect("context").calls; + assert_eq!( + calls + .iter() + .map(|call| (call.requested_at, call.after_compaction)) + .collect::>(), + [ + (at("2026-09-01T10:00:01Z"), false), + (at("2026-09-01T10:00:12Z"), false), + (None, true), + ] + ); + assert_eq!(calls[1].cache_read_tokens, Some(29_952)); + assert_eq!(calls[1].input_tokens, Some(2_048)); + assert_eq!(calls[1].cache_creation_tokens, Some(0)); + assert_eq!(calls[2].completed_at, at("2026-09-01T10:00:30Z").unwrap()); +} + +#[test] +fn an_oversized_compaction_row_still_marks_the_boundary() { + let compaction = format!( + r#"{{"timestamp":"2026-09-01T10:00:21Z","type":"response_item","payload":{{"type":"compaction","encrypted_content":"{}"}}}}"#, + "x".repeat(MAX_JSONL_RECORD_BYTES + 1) + ); + let scan = scan_lines(&turn_lines(compaction)); + + assert_eq!(scan.diagnostics.oversized_rows, 1); + let calls = &scan.events[0].context.as_ref().expect("context").calls; + assert!(calls[2].after_compaction); +} + +#[test] +fn token_counts_alone_do_not_place_request_starts() { + let lines = vec![ + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"session_meta","payload":{"id":"thread-1"}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:01Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"go"}]}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:12Z","type":"response_item","payload":{"type":"function_call_output","call_id":"c1","output":"ok"}}"#.to_string(), + token_count("2026-09-01T10:00:12.500Z", 30_050, 30_000, 0), + r#"{"timestamp":"2026-09-01T10:00:30Z","type":"response_item","payload":{"type":"function_call_output","call_id":"c2","output":"ok"}}"#.to_string(), + token_count("2026-09-01T10:00:30.500Z", 62_150, 32_000, 29_952), + ]; + let scan = scan_lines(&lines); + + assert_eq!(scan.events.len(), 2); + assert!(scan.events.iter().all(|event| event.context.is_none())); +} + +#[test] +fn a_usage_record_outside_a_turn_carries_its_request_start() { + let lines = vec![ + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"session_meta","payload":{"id":"thread-1"}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:01Z","type":"event_msg","payload":{"type":"user_message","message":"go"}}"#.to_string(), + usage_record("2026-09-01T10:00:10Z", 30_000, 0), + ]; + let scan = scan_lines(&lines); + + assert_eq!(scan.events.len(), 1); + let context = scan.events[0].context.as_ref().expect("context"); + assert_eq!(context.requested_at, at("2026-09-01T10:00:01Z")); + assert!(context.calls.is_empty()); +} + +#[test] +fn turn_calls_carry_the_model_that_served_each() { + let scan = scan_lines(&[ + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"session_meta","payload":{"id":"thread-1"}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"event_msg","payload":{"type":"task_started","turn_id":"turn-1"}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"turn_context","payload":{"model":"gpt-5"}}"#.to_string(), + usage_record("2026-09-01T10:00:10Z", 30_000, 0), + r#"{"timestamp":"2026-09-01T10:00:11Z","type":"turn_context","payload":{"model":"gpt-5.1"}}"#.to_string(), + usage_record("2026-09-01T10:00:20Z", 30_000, 0), + r#"{"timestamp":"2026-09-01T10:00:40Z","type":"event_msg","payload":{"type":"task_complete","turn_id":"turn-1"}}"#.to_string(), + ]); + let calls = &scan.events[0].context.as_ref().expect("context").calls; + assert_eq!( + calls + .iter() + .map(|call| call.model.as_deref()) + .collect::>(), + [Some("gpt-5"), Some("gpt-5.1")] + ); + + // A model the rollout never named is a fallback guess, not evidence. + let scan = scan_lines(&turn_lines(String::new())); + let calls = &scan.events[0].context.as_ref().expect("context").calls; + assert!(calls.iter().all(|call| call.model.is_none())); +} + +#[test] +fn completion_usage_keeps_the_turns_compaction() { + let completion = |timestamp: &str| { + format!( + r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"task_complete","completed_at":"{timestamp}","duration_ms":3000}},"usage":{{"input_tokens":30000,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0,"total_tokens":30050}}}}"# + ) + }; + let compacted = r#"{"timestamp":"2026-09-01T10:00:05Z","type":"compacted","payload":{"message":"","replacement_history":[]}}"#; + let opening = [ + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"session_meta","payload":{"id":"thread-1"}}"#, + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"event_msg","payload":{"type":"task_started","turn_id":"turn-1"}}"#, + ] + .map(str::to_string); + + // The completion's own usage is the call after the compaction. + let mut lines = opening.to_vec(); + lines.extend([compacted.to_string(), completion("2026-09-01T10:00:10Z")]); + let scan = scan_lines(&lines); + assert_eq!(scan.events.len(), 1); + assert!( + scan.events[0] + .context + .as_ref() + .expect("context") + .after_compaction + ); + + // A usage line inside the turn took the mark; the completion replaces it. + let mut lines = opening.to_vec(); + lines.extend([ + compacted.to_string(), + usage_record("2026-09-01T10:00:08Z", 30_000, 0), + completion("2026-09-01T10:00:10Z"), + ]); + let scan = scan_lines(&lines); + assert_eq!(scan.events.len(), 1); + assert!( + scan.events[0] + .context + .as_ref() + .expect("context") + .after_compaction + ); +} + +#[test] +fn a_call_without_input_counts_keeps_them_unreported() { + let scan = scan_lines(&[ + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"session_meta","payload":{"id":"thread-1"}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"event_msg","payload":{"type":"task_started","turn_id":"turn-1"}}"#.to_string(), + usage_record("2026-09-01T10:00:10Z", 30_000, 0), + r#"{"timestamp":"2026-09-01T10:00:20Z","type":"token_usage_record","payload":{"usage":{"output_tokens":50,"total_tokens":50}}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:40Z","type":"event_msg","payload":{"type":"task_complete","turn_id":"turn-1"}}"#.to_string(), + ]); + let event = &scan.events[0]; + let calls = &event.context.as_ref().expect("context").calls; + assert_eq!(calls.len(), 2); + assert_eq!(calls[1].input_tokens, None); + assert_eq!(calls[1].cache_read_tokens, None); + + let analyzed = statsai_core::analyze_cache_calls([event]); + assert_eq!( + analyzed[1].verdict.unclassifiable, + Some(statsai_core::CacheUnclassifiableReason::NoInputTelemetry) + ); +} diff --git a/crates/statsai-adapters/src/codex/tests/parse/mod.rs b/crates/statsai-adapters/src/codex/tests/parse/mod.rs index f7f42f0..708e9cc 100644 --- a/crates/statsai-adapters/src/codex/tests/parse/mod.rs +++ b/crates/statsai-adapters/src/codex/tests/parse/mod.rs @@ -1,5 +1,6 @@ pub(crate) use super::*; +mod calls; mod session; mod usage; diff --git a/crates/statsai-adapters/src/event.rs b/crates/statsai-adapters/src/event.rs index f38a812..6653247 100644 --- a/crates/statsai-adapters/src/event.rs +++ b/crates/statsai-adapters/src/event.rs @@ -402,6 +402,7 @@ pub(crate) fn usage_event( privacy: metadata_only_privacy(), created_at: parts.timestamp, imported_at: Utc::now(), + context: None, } } @@ -491,6 +492,7 @@ pub(crate) fn runtime_to_summary_metrics(runtime: RuntimeInfo) -> SummaryMetrics user_messages: runtime.user_messages, assistant_messages: runtime.assistant_messages, developer_messages: runtime.developer_messages, + cache_health: None, } } diff --git a/crates/statsai-adapters/src/grok/parse/mod.rs b/crates/statsai-adapters/src/grok/parse/mod.rs index d575f54..cac5795 100644 --- a/crates/statsai-adapters/src/grok/parse/mod.rs +++ b/crates/statsai-adapters/src/grok/parse/mod.rs @@ -330,6 +330,7 @@ pub(crate) fn parse_grok_summary( user_messages, assistant_messages, developer_messages: None, + cache_health: None, }); } scan.diagnostics.raw_rows += stats diff --git a/crates/statsai-adapters/src/json.rs b/crates/statsai-adapters/src/json.rs index ee8f0bf..4aeff20 100644 --- a/crates/statsai-adapters/src/json.rs +++ b/crates/statsai-adapters/src/json.rs @@ -8,6 +8,10 @@ use std::io::{BufReader, Cursor}; pub(crate) const MAX_JSONL_RECORD_BYTES: usize = 16 * 1024 * 1024; +/// Leading bytes of an oversized record kept for classification. Enough for +/// the record type and timestamp, which providers write first. +pub(crate) const OVERSIZED_RECORD_HEADER_BYTES: usize = 256; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum BoundedLineRead { Eof, @@ -30,7 +34,7 @@ pub(crate) fn read_bounded_jsonl_line( return Ok(BoundedLineRead::Eof); } if oversized || line.len() > max_bytes { - line.clear(); + line.truncate(oversized_header_len(max_bytes)); return Ok(BoundedLineRead::Oversized); } return Ok(BoundedLineRead::Complete); @@ -62,7 +66,10 @@ pub(crate) fn read_bounded_jsonl_line( if record_bytes <= max_bytes || deferred_cr { line.extend_from_slice(&available[..consumed]); } else { - line.clear(); + let header_len = oversized_header_len(max_bytes); + line.truncate(header_len); + let missing = header_len - line.len(); + line.extend_from_slice(&available[..consumed.min(missing)]); oversized = true; } } @@ -77,6 +84,10 @@ pub(crate) fn read_bounded_jsonl_line( } } +fn oversized_header_len(max_bytes: usize) -> usize { + OVERSIZED_RECORD_HEADER_BYTES.min(max_bytes) +} + pub(crate) fn number_at_any(value: &Value, keys: &[&str]) -> Option { keys.iter() .find_map(|key| value.get(*key).and_then(value_as_u64)) @@ -168,7 +179,7 @@ fn bounded_jsonl_reader_discards_an_oversized_record_and_recovers_next_line() { read_bounded_jsonl_line(&mut reader, &mut line, 8).expect("oversized line"), BoundedLineRead::Oversized ); - assert!(line.is_empty()); + assert_eq!(line, b"xxxxxxxx", "keeps the record's leading bytes"); assert_eq!( read_bounded_jsonl_line(&mut reader, &mut line, 32).expect("next line"), BoundedLineRead::Complete @@ -209,5 +220,5 @@ fn bounded_jsonl_reader_handles_crlf_split_across_buffers_at_the_limit() { .expect("unterminated trailing CR"), BoundedLineRead::Oversized ); - assert!(line.is_empty()); + assert_eq!(line, b"12345678"); } diff --git a/crates/statsai-core/src/cache/detector.rs b/crates/statsai-core/src/cache/detector.rs new file mode 100644 index 0000000..1075597 --- /dev/null +++ b/crates/statsai-core/src/cache/detector.rs @@ -0,0 +1,554 @@ +use super::{ + provider_has_cache_diagnostics, CACHE_LOSS_MIN_PERCENT, CACHE_LOSS_MIN_TOKENS, + CACHE_WRITE_REPORTING_PROVIDERS, +}; +use crate::ids::EventId; +use crate::types::{ModelInfo, UsageEvent}; +use chrono::{DateTime, Utc}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// How the detector classified one model call. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CacheCallClass { + /// The first call observed in its stream. Nothing earlier to compare with. + FirstObserved, + /// The comparison restarts here; see [`CacheBoundaryReason`]. + Boundary, + /// The previous call left too little reusable context to judge a loss. + BelowThreshold, + /// Compared with the previous call in its stream. + Comparable, + /// The call neither read nor wrote the cache, so it did not use it, as + /// when a custom endpoint serves the request. Not judged, and it leaves + /// nothing reusable for the next call. + Uncached, + /// Usage that cannot be placed in order or split into calls. + Unclassifiable, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CacheBoundaryReason { + /// The model changed. Prompt caches are per model. + ModelChange, + /// The provider compacted the conversation, which rewrites the prompt. + Compaction, + /// The previous usage in this stream could not be classified. + EvidenceGap, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CacheUnclassifiableReason { + /// The record had no timestamp, so its place in the stream is unknown. + TimestampMissing, + /// Several calls were recorded as one total with no per-call breakdown. + Aggregated, + /// The provider reported no input or cache counts. + NoInputTelemetry, + /// Another call in the stream has the same timestamp and no recorded order. + AmbiguousOrder, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CacheLoss { + /// No cached reads remained. + Full, + /// Some cached context was still read. + Partial, +} + +/// What a request gap was measured between. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CacheGapTiming { + /// From the previous response to the record that sent this request. + RequestStart, + /// Between the two recorded responses. Includes this call's own run time, + /// so it overstates idle time. + RecordedEvents, +} + +/// The cache lifetime a call's writes asked for, as Claude Code records it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CacheWriteTtl { + FiveMinutes, + OneHour, + Mixed, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct CacheCallVerdict { + pub class: CacheCallClass, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub boundary: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub unclassifiable: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub loss: Option, + /// Reusable context the previous call left, bounded by this call's input. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub baseline_tokens: Option, + /// For a suspected loss, the reusable context it did not read from the + /// cache: the baseline minus its cached reads. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub missed_tokens: Option, + /// What the missed reuse cost at API prices: the call's cost minus its + /// cost had the missed tokens been cache reads. Set by the caller that + /// prices calls; `None` when the model's pricing is unknown. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub missed_cost_micro_usd: Option, + /// Seconds since the previous call in the stream completed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gap_seconds: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gap_timing: Option, + /// The model before a model change. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub previous_model: Option, + /// Cache lifetime the previous call's writes asked for, when recorded. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub previous_write_ttl: Option, +} + +impl CacheCallVerdict { + fn new(class: CacheCallClass) -> Self { + Self { + class, + boundary: None, + unclassifiable: None, + loss: None, + baseline_tokens: None, + missed_tokens: None, + missed_cost_micro_usd: None, + gap_seconds: None, + gap_timing: None, + previous_model: None, + previous_write_ttl: None, + } + } + + fn unclassifiable(reason: CacheUnclassifiableReason) -> Self { + Self { + unclassifiable: Some(reason), + ..Self::new(CacheCallClass::Unclassifiable) + } + } +} + +/// One model call and its verdict, or one unclassifiable block of usage. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct AnalyzedCacheCall { + pub event_id: EventId, + /// Position inside an aggregated event; zero for single-call events. + pub call_index: u32, + /// Model calls this entry stands for: one, or an aggregated event's count. + pub requests: u64, + pub provider: String, + pub source_id: String, + pub provider_account_id: Option, + pub session_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub agent_hash: Option, + pub completed_at: DateTime, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub requested_at: Option>, + pub model: Option, + pub input_tokens: u64, + pub cache_read_tokens: u64, + pub cache_creation_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub write_ttl: Option, + pub verdict: CacheCallVerdict, +} + +impl AnalyzedCacheCall { + /// Ordinary input plus cache writes and reads. + #[must_use] + pub fn logical_input_tokens(&self) -> u64 { + self.input_tokens + .saturating_add(self.cache_creation_tokens.unwrap_or(0)) + .saturating_add(self.cache_read_tokens) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct StreamKey { + provider: String, + source_id: String, + provider_account_id: Option, + session_id: String, + file_hash: Option, + agent_hash: Option, +} + +/// Classifies every model call of the given events. +/// +/// Pass whole sessions: a call is judged against its predecessor, and a +/// predecessor that is not passed in makes the call look like the first one. +/// Events of providers without call order are skipped. The result is sorted by +/// completion time, then event and call position, so equal input always gives +/// equal output. +#[must_use] +pub fn analyze_cache_calls<'a>( + events: impl IntoIterator, +) -> Vec { + let mut streams: BTreeMap> = BTreeMap::new(); + let mut analyzed = Vec::new(); + for event in events { + if !provider_has_cache_diagnostics(&event.provider) { + continue; + } + let calls = event_calls(event); + let timestamp_missing = event + .parse_evidence + .as_ref() + .is_some_and(|evidence| evidence.timestamp_inferred); + if timestamp_missing { + // Its place in the stream is unknown, so it neither gets compared + // nor interrupts the calls around it. + analyzed.push(unclassifiable_entry( + event, + CacheUnclassifiableReason::TimestampMissing, + )); + continue; + } + let line = event + .parse_evidence + .as_ref() + .and_then(|evidence| evidence.source_line_number); + let stream = streams.entry(stream_key(event)).or_default(); + for call in calls { + let order = OrderKey { + completed_at: call.completed_at, + line, + event_id: call.event_id.0.clone(), + call_index: call.call_index, + }; + stream.push((order, call)); + } + } + for mut stream in streams.into_values() { + stream.sort_by(|left, right| left.0.cmp(&right.0)); + mark_ambiguous_order(&mut stream); + classify_stream(&mut stream); + analyzed.extend(stream.into_iter().map(|(_, call)| call)); + } + analyzed.sort_by(|left, right| { + left.completed_at + .cmp(&right.completed_at) + .then_with(|| left.event_id.0.cmp(&right.event_id.0)) + .then_with(|| left.call_index.cmp(&right.call_index)) + }); + analyzed +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct OrderKey { + completed_at: DateTime, + line: Option, + event_id: String, + call_index: u32, +} + +fn stream_key(event: &UsageEvent) -> StreamKey { + StreamKey { + provider: event.provider.clone(), + source_id: event.source_id.0.clone(), + provider_account_id: event.provider_account_id.as_ref().map(|id| id.0.clone()), + session_id: event.session.session_id.clone(), + file_hash: event + .parse_evidence + .as_ref() + .and_then(|evidence| evidence.source_file_path_hash.clone()), + agent_hash: event + .context + .as_ref() + .and_then(|context| context.agent_hash.clone()), + } +} + +/// The name the detector compares to tell whether consecutive calls used the +/// same model, and so the same prompt cache. +#[must_use] +pub fn cache_model_name(model: &ModelInfo) -> Option { + model + .normalized_name + .clone() + .or_else(|| model.name.clone()) + .filter(|name| !name.trim().is_empty()) +} + +/// The event's model, unless the parser filled in a fallback because the +/// transcript did not name one: a guess cannot show that the model changed. +fn event_model(event: &UsageEvent) -> Option { + let inferred = event + .parse_evidence + .as_ref() + .is_some_and(|evidence| evidence.model_inferred); + event + .model + .as_ref() + .filter(|_| !inferred) + .and_then(cache_model_name) +} + +fn base_entry(event: &UsageEvent, verdict: CacheCallVerdict) -> AnalyzedCacheCall { + AnalyzedCacheCall { + event_id: event.event_id.clone(), + call_index: 0, + requests: 1, + provider: event.provider.clone(), + source_id: event.source_id.0.clone(), + provider_account_id: event.provider_account_id.as_ref().map(|id| id.0.clone()), + session_id: event.session.session_id.clone(), + agent_hash: event + .context + .as_ref() + .and_then(|context| context.agent_hash.clone()), + completed_at: event.created_at, + requested_at: None, + model: event_model(event), + input_tokens: event.usage.input_tokens.unwrap_or(0), + cache_read_tokens: event.usage.cache_read_tokens.unwrap_or(0), + cache_creation_tokens: event.usage.cache_creation_tokens, + write_ttl: None, + verdict, + } +} + +fn unclassifiable_entry( + event: &UsageEvent, + reason: CacheUnclassifiableReason, +) -> AnalyzedCacheCall { + AnalyzedCacheCall { + requests: event.usage.requests.unwrap_or(1).max(1), + ..base_entry(event, CacheCallVerdict::unclassifiable(reason)) + } +} + +/// The calls an event stands for, before classification. An event that cannot +/// be split into calls comes back as one unclassifiable entry. +fn event_calls(event: &UsageEvent) -> Vec { + let requests = event.usage.requests.unwrap_or(1).max(1); + let recorded_calls = event + .context + .as_ref() + .map(|context| context.calls.as_slice()) + .unwrap_or_default(); + if !recorded_calls.is_empty() { + if recorded_calls.len() as u64 != requests { + return vec![unclassifiable_entry( + event, + CacheUnclassifiableReason::Aggregated, + )]; + } + return recorded_calls + .iter() + .enumerate() + .map(|(index, call)| { + let verdict = if call.input_tokens.is_none() && call.cache_read_tokens.is_none() { + CacheCallVerdict::unclassifiable(CacheUnclassifiableReason::NoInputTelemetry) + } else { + CacheCallVerdict { + boundary: call + .after_compaction + .then_some(CacheBoundaryReason::Compaction), + ..CacheCallVerdict::new(CacheCallClass::Comparable) + } + }; + AnalyzedCacheCall { + call_index: index as u32, + completed_at: call.completed_at, + requested_at: call.requested_at, + input_tokens: call.input_tokens.unwrap_or(0), + cache_read_tokens: call.cache_read_tokens.unwrap_or(0), + cache_creation_tokens: call.cache_creation_tokens, + // A turn's calls can switch models, so the event's model, + // which is the turn's last, does not stand in for each. + model: call.model.clone(), + verdict, + ..base_entry(event, CacheCallVerdict::new(CacheCallClass::Comparable)) + } + }) + .collect(); + } + if requests > 1 { + return vec![unclassifiable_entry( + event, + CacheUnclassifiableReason::Aggregated, + )]; + } + if event.usage.input_tokens.is_none() && event.usage.cache_read_tokens.is_none() { + return vec![unclassifiable_entry( + event, + CacheUnclassifiableReason::NoInputTelemetry, + )]; + } + let context = event.context.as_ref(); + let compaction = context.is_some_and(|context| context.after_compaction); + vec![AnalyzedCacheCall { + requested_at: context.and_then(|context| context.requested_at), + write_ttl: write_ttl( + event.usage.cache_creation_5m_tokens, + event.usage.cache_creation_1h_tokens, + ), + verdict: CacheCallVerdict { + boundary: compaction.then_some(CacheBoundaryReason::Compaction), + ..CacheCallVerdict::new(CacheCallClass::Comparable) + }, + ..base_entry(event, CacheCallVerdict::new(CacheCallClass::Comparable)) + }] +} + +fn write_ttl(five_minutes: Option, one_hour: Option) -> Option { + match (five_minutes.unwrap_or(0) > 0, one_hour.unwrap_or(0) > 0) { + (true, true) => Some(CacheWriteTtl::Mixed), + (true, false) => Some(CacheWriteTtl::FiveMinutes), + (false, true) => Some(CacheWriteTtl::OneHour), + (false, false) => None, + } +} + +/// Calls with one timestamp keep their order only when every two of them carry +/// source lines or belong to the same event. Otherwise none of them is +/// compared: any could have come first, so no member's predecessor is known. +fn mark_ambiguous_order(stream: &mut [(OrderKey, AnalyzedCacheCall)]) { + let mut start = 0; + while start < stream.len() { + let completed_at = stream[start].0.completed_at; + let end = start + + stream[start..] + .iter() + .take_while(|(order, _)| order.completed_at == completed_at) + .count(); + let group = &mut stream[start..end]; + let ordered = group.iter().enumerate().all(|(index, (left, _))| { + group[index + 1..].iter().all(|(right, _)| { + left.event_id == right.event_id || (left.line.is_some() && right.line.is_some()) + }) + }); + if !ordered { + for (_, call) in group.iter_mut() { + if call.verdict.class != CacheCallClass::Unclassifiable { + call.verdict = + CacheCallVerdict::unclassifiable(CacheUnclassifiableReason::AmbiguousOrder); + } + } + } + start = end; + } +} + +struct Previous { + completed_at: DateTime, + reusable_tokens: u64, + model: Option, + write_ttl: Option, +} + +fn classify_stream(stream: &mut [(OrderKey, AnalyzedCacheCall)]) { + let mut started = false; + let mut after_gap = false; + let mut previous: Option = None; + for (_, call) in stream.iter_mut() { + if call.verdict.class == CacheCallClass::Unclassifiable { + started = true; + after_gap = true; + previous = None; + continue; + } + let compaction = call.verdict.boundary == Some(CacheBoundaryReason::Compaction); + let mut verdict = CacheCallVerdict::new(CacheCallClass::Comparable); + if let Some(previous) = previous.as_ref() { + let (gap_seconds, gap_timing) = gap(previous.completed_at, call); + verdict.gap_seconds = Some(gap_seconds); + verdict.gap_timing = Some(gap_timing); + verdict.previous_write_ttl = previous.write_ttl; + } + let model_changed = previous.as_ref().is_some_and(|previous| { + previous.model.is_some() && call.model.is_some() && previous.model != call.model + }); + if skipped_the_cache(call) { + verdict.class = CacheCallClass::Uncached; + } else if !started { + verdict.class = CacheCallClass::FirstObserved; + } else if after_gap { + verdict.class = CacheCallClass::Boundary; + verdict.boundary = Some(CacheBoundaryReason::EvidenceGap); + } else if compaction { + verdict.class = CacheCallClass::Boundary; + verdict.boundary = Some(CacheBoundaryReason::Compaction); + } else if model_changed { + verdict.class = CacheCallClass::Boundary; + verdict.boundary = Some(CacheBoundaryReason::ModelChange); + verdict.previous_model = previous + .as_ref() + .and_then(|previous| previous.model.clone()); + } else if let Some(previous) = previous.as_ref() { + let baseline = previous.reusable_tokens.min(call.logical_input_tokens()); + verdict.baseline_tokens = Some(baseline); + if baseline < CACHE_LOSS_MIN_TOKENS { + verdict.class = CacheCallClass::BelowThreshold; + } else { + verdict.loss = loss(baseline, call.cache_read_tokens); + verdict.missed_tokens = verdict + .loss + .map(|_| baseline.saturating_sub(call.cache_read_tokens)); + } + } + call.verdict = verdict; + started = true; + after_gap = false; + previous = Some(Previous { + completed_at: call.completed_at, + reusable_tokens: call + .cache_read_tokens + .saturating_add(call.cache_creation_tokens.unwrap_or(0)), + model: call.model.clone(), + write_ttl: call.write_ttl, + }); + } +} + +/// Whether a call reported no cache reads and no cache writes, for a provider +/// whose calls report writes whenever they use the cache. +fn skipped_the_cache(call: &AnalyzedCacheCall) -> bool { + CACHE_WRITE_REPORTING_PROVIDERS.contains(&call.provider.as_str()) + && call.input_tokens > 0 + && call.cache_read_tokens == 0 + && call.cache_creation_tokens == Some(0) +} + +/// Seconds from the previous response to this request. A recorded request +/// start that precedes the previous response belongs to an earlier exchange, +/// so the gap falls back to the two responses' own timestamps. +fn gap(previous_completed_at: DateTime, call: &AnalyzedCacheCall) -> (u64, CacheGapTiming) { + let seconds = |from: DateTime, to: DateTime| (to - from).num_seconds().max(0) as u64; + match call.requested_at { + Some(requested_at) if requested_at >= previous_completed_at => ( + seconds(previous_completed_at, requested_at), + CacheGapTiming::RequestStart, + ), + _ => ( + seconds(previous_completed_at, call.completed_at), + CacheGapTiming::RecordedEvents, + ), + } +} + +fn loss(baseline: u64, cache_read_tokens: u64) -> Option { + let drop = baseline.saturating_sub(cache_read_tokens); + let suspected = drop >= CACHE_LOSS_MIN_TOKENS + && u128::from(drop) * 100 >= u128::from(baseline) * u128::from(CACHE_LOSS_MIN_PERCENT); + suspected.then_some(if cache_read_tokens == 0 { + CacheLoss::Full + } else { + CacheLoss::Partial + }) +} diff --git a/crates/statsai-core/src/cache/health.rs b/crates/statsai-core/src/cache/health.rs new file mode 100644 index 0000000..779019f --- /dev/null +++ b/crates/statsai-core/src/cache/health.rs @@ -0,0 +1,246 @@ +use super::{ + cache_gap_bin, AnalyzedCacheCall, CacheBoundaryReason, CacheCallClass, CacheGapTiming, + CacheLoss, CACHE_GAP_BIN_COUNT, CACHE_HEALTH_VERSION, +}; +use crate::types::UsageEvent; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// Bounded prompt-cache diagnostics for one daily summary. +/// +/// Only counts leave the device: no timestamps, ids, or per-call values. Token +/// totals stay on the summary's `usage`, so this adds only what they cannot +/// say. `requests` equals `unclassifiable + first_observed + boundaries + +/// below_threshold + uncached + comparable`, and the gap bins add up to +/// `comparable`, `losses`, and `missed_tokens`. Monthly rollups embed every +/// day's object, so the fields most days leave at zero are left out then. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct CacheHealthV1 { + pub version: u32, + /// Model calls of providers the detector covers. + pub requests: u64, + pub unclassifiable: u64, + /// First calls of their stream. + pub first_observed: u64, + /// First calls that read nothing from the cache. + pub first_cold: u64, + /// Calls after a model change, compaction, or unclassifiable usage. + pub boundaries: u64, + /// Calls whose predecessor left too little reusable context to judge. + pub below_threshold: u64, + /// Calls that neither read nor wrote the cache, so did not use it. + #[serde(default, skip_serializing_if = "is_zero")] + pub uncached: u64, + pub comparable: u64, + /// Suspected cache losses among comparable calls. + pub losses: u64, + /// Suspected losses that read nothing from the cache. + pub full_losses: u64, + /// Reusable context suspected losses did not read from the cache. + #[serde(default, skip_serializing_if = "is_zero")] + pub missed_tokens: u64, + /// What that missed reuse cost at API prices, in micro-USD. + #[serde(default, skip_serializing_if = "is_zero")] + pub missed_cost_micro_usd: u64, + /// Suspected losses on models without known pricing, left out of + /// `missed_cost_micro_usd`. + #[serde(default, skip_serializing_if = "is_zero")] + pub unpriced_losses: u64, + /// Comparable calls whose gap was measured between recorded responses + /// rather than from a recorded request start. + pub timing_estimated: u64, + /// Ordinary input from calls whose provider did not report cache writes, + /// so it may include writes. + pub write_split_unavailable_input_tokens: u64, + /// Comparable calls by gap since the previous call, one count per bin of + /// [`super::CACHE_GAP_BIN_UPPER_MINUTES`]: `<1m` first, `60m+` last. + pub gap_comparable: Vec, + /// Suspected losses in the same bins. + pub gap_losses: Vec, + /// Missed tokens in the same bins. + #[serde(default = "empty_gap_bins", skip_serializing_if = "all_zero")] + pub gap_missed_tokens: Vec, +} + +fn is_zero(value: &u64) -> bool { + *value == 0 +} + +fn all_zero(values: &[u64]) -> bool { + values.iter().all(|value| *value == 0) +} + +fn empty_gap_bins() -> Vec { + vec![0; CACHE_GAP_BIN_COUNT] +} + +impl Default for CacheHealthV1 { + fn default() -> Self { + Self { + version: CACHE_HEALTH_VERSION, + requests: 0, + unclassifiable: 0, + first_observed: 0, + first_cold: 0, + boundaries: 0, + below_threshold: 0, + uncached: 0, + comparable: 0, + losses: 0, + full_losses: 0, + missed_tokens: 0, + missed_cost_micro_usd: 0, + unpriced_losses: 0, + timing_estimated: 0, + write_split_unavailable_input_tokens: 0, + gap_comparable: empty_gap_bins(), + gap_losses: empty_gap_bins(), + gap_missed_tokens: empty_gap_bins(), + } + } +} + +impl CacheHealthV1 { + /// Calls the detector could place and classify. + #[must_use] + pub fn analyzed(&self) -> u64 { + self.requests.saturating_sub(self.unclassifiable) + } + + pub fn add_call(&mut self, call: &AnalyzedCacheCall) { + self.requests = self.requests.saturating_add(call.requests); + let verdict = &call.verdict; + match verdict.class { + CacheCallClass::Unclassifiable => { + self.unclassifiable = self.unclassifiable.saturating_add(call.requests); + } + CacheCallClass::FirstObserved => { + self.first_observed = self.first_observed.saturating_add(1); + if call.cache_read_tokens == 0 { + self.first_cold = self.first_cold.saturating_add(1); + } + } + CacheCallClass::Boundary => self.boundaries = self.boundaries.saturating_add(1), + CacheCallClass::BelowThreshold => { + self.below_threshold = self.below_threshold.saturating_add(1); + } + CacheCallClass::Uncached => self.uncached = self.uncached.saturating_add(1), + CacheCallClass::Comparable => { + self.comparable = self.comparable.saturating_add(1); + if verdict.gap_timing == Some(CacheGapTiming::RecordedEvents) { + self.timing_estimated = self.timing_estimated.saturating_add(1); + } + let lost = verdict.loss.is_some(); + if lost { + self.losses = self.losses.saturating_add(1); + } + if verdict.loss == Some(CacheLoss::Full) { + self.full_losses = self.full_losses.saturating_add(1); + } + match (lost, verdict.missed_cost_micro_usd) { + (true, Some(cost)) => { + self.missed_cost_micro_usd = + self.missed_cost_micro_usd.saturating_add(cost); + } + (true, None) => self.unpriced_losses = self.unpriced_losses.saturating_add(1), + (false, _) => {} + } + let missed = verdict.missed_tokens.unwrap_or(0); + self.missed_tokens = self.missed_tokens.saturating_add(missed); + if let Some(bin) = verdict.gap_seconds.map(cache_gap_bin) { + if let Some(count) = self.gap_comparable.get_mut(bin) { + *count = count.saturating_add(1); + } + if let Some(count) = self.gap_losses.get_mut(bin).filter(|_| lost) { + *count = count.saturating_add(1); + } + if let Some(tokens) = self.gap_missed_tokens.get_mut(bin) { + *tokens = tokens.saturating_add(missed); + } + } + } + } + } + + /// Counts ordinary input as unsplit when its provider reported no cache + /// writes. An event that lists its calls is counted call by call: summing + /// one reported zero with unreported writes would hide the rest. + pub fn add_event_input(&mut self, event: &UsageEvent) { + let calls = event + .context + .as_ref() + .map(|context| context.calls.as_slice()) + .unwrap_or_default(); + let unsplit = if !calls.is_empty() { + calls + .iter() + .filter(|call| call.cache_creation_tokens.is_none()) + .map(|call| call.input_tokens.unwrap_or(0)) + .fold(0, u64::saturating_add) + } else if event.usage.cache_creation_tokens.is_none() { + event.usage.input_tokens.unwrap_or(0) + } else { + 0 + }; + self.write_split_unavailable_input_tokens = self + .write_split_unavailable_input_tokens + .saturating_add(unsplit); + } + + pub fn merge(&mut self, other: &Self) { + self.requests = self.requests.saturating_add(other.requests); + self.unclassifiable = self.unclassifiable.saturating_add(other.unclassifiable); + self.first_observed = self.first_observed.saturating_add(other.first_observed); + self.first_cold = self.first_cold.saturating_add(other.first_cold); + self.boundaries = self.boundaries.saturating_add(other.boundaries); + self.below_threshold = self.below_threshold.saturating_add(other.below_threshold); + self.uncached = self.uncached.saturating_add(other.uncached); + self.comparable = self.comparable.saturating_add(other.comparable); + self.losses = self.losses.saturating_add(other.losses); + self.full_losses = self.full_losses.saturating_add(other.full_losses); + self.missed_tokens = self.missed_tokens.saturating_add(other.missed_tokens); + self.missed_cost_micro_usd = self + .missed_cost_micro_usd + .saturating_add(other.missed_cost_micro_usd); + self.unpriced_losses = self.unpriced_losses.saturating_add(other.unpriced_losses); + self.timing_estimated = self.timing_estimated.saturating_add(other.timing_estimated); + self.write_split_unavailable_input_tokens = self + .write_split_unavailable_input_tokens + .saturating_add(other.write_split_unavailable_input_tokens); + for (count, other) in self.gap_comparable.iter_mut().zip(&other.gap_comparable) { + *count = count.saturating_add(*other); + } + for (count, other) in self.gap_losses.iter_mut().zip(&other.gap_losses) { + *count = count.saturating_add(*other); + } + for (tokens, other) in self + .gap_missed_tokens + .iter_mut() + .zip(&other.gap_missed_tokens) + { + *tokens = tokens.saturating_add(*other); + } + } +} + +/// Boundary counts by reason. Local only: the synced object carries the total. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct CacheBoundaryCounts { + pub model_change: u64, + pub compaction: u64, + pub evidence_gap: u64, +} + +impl CacheBoundaryCounts { + pub fn add_call(&mut self, call: &AnalyzedCacheCall) { + if call.verdict.class != CacheCallClass::Boundary { + return; + } + let count = match call.verdict.boundary { + Some(CacheBoundaryReason::ModelChange) => &mut self.model_change, + Some(CacheBoundaryReason::Compaction) => &mut self.compaction, + Some(CacheBoundaryReason::EvidenceGap) | None => &mut self.evidence_gap, + }; + *count = count.saturating_add(1); + } +} diff --git a/crates/statsai-core/src/cache/mod.rs b/crates/statsai-core/src/cache/mod.rs new file mode 100644 index 0000000..e416d10 --- /dev/null +++ b/crates/statsai-core/src/cache/mod.rs @@ -0,0 +1,97 @@ +//! Prompt-cache diagnostics. +//! +//! The report answers three questions for a period: how much logical input was +//! served from the provider's prompt cache, when that reuse dropped, and how +//! much context the model calls processed regardless. The first and last come +//! straight from token totals. The second needs request order, so it runs a +//! conservative detector over each stream of model calls. +//! +//! A stream is one agent's calls within one session: the same provider, source, +//! account, session, transcript file, and sub-agent. Each call is compared only +//! with the call before it in its stream. The previous call leaves its cache +//! reads plus its known cache writes reusable; the current call is expected to +//! read about that much again, bounded by its own input size. When reads fall at +//! least half and at least [`CACHE_LOSS_MIN_TOKENS`] below that baseline, the +//! call is a suspected cache loss. That is a heuristic over recorded token +//! counts, not a provider-confirmed miss. +//! +//! Comparisons restart, rather than count as losses, at the first call of a +//! stream, after a model change, after compaction, and after a call whose +//! usage cannot be placed or split into calls. A call that neither read nor +//! wrote the cache did not use it, as when a custom endpoint serves the +//! request, so it is counted apart and not judged. + +mod detector; +mod health; +mod report; + +pub use detector::*; +pub use health::*; +pub use report::*; + +/// Version of the detector rules and of the synced `cache_health` object. +pub const CACHE_HEALTH_VERSION: u32 = 1; + +/// Smallest drop in cached reads that can count as a loss. +pub const CACHE_LOSS_MIN_TOKENS: u64 = 4_096; + +/// A loss also needs reads to fall by at least this share of the baseline. +pub const CACHE_LOSS_MIN_PERCENT: u64 = 50; + +/// Upper bounds, in minutes, of the request-gap bins. The last bin is open. +pub const CACHE_GAP_BIN_UPPER_MINUTES: [u64; 7] = [1, 5, 10, 15, 30, 45, 60]; + +/// Number of request-gap bins, including the open-ended last one. +pub const CACHE_GAP_BIN_COUNT: usize = CACHE_GAP_BIN_UPPER_MINUTES.len() + 1; + +/// Providers whose usage events record calls in order, which the detector +/// needs. Token totals for every other provider still count toward hit rates. +pub const CACHE_DIAGNOSTIC_PROVIDERS: [&str; 2] = ["claude_code", "codex"]; + +#[must_use] +pub fn provider_has_cache_diagnostics(provider: &str) -> bool { + CACHE_DIAGNOSTIC_PROVIDERS.contains(&provider) +} + +/// Providers that report cache writes on every call that uses the cache, so a +/// call reporting no reads and no writes did not use it. Codex reports zero +/// writes on calls that do, so it is not one of them. +pub const CACHE_WRITE_REPORTING_PROVIDERS: [&str; 1] = ["claude_code"]; + +/// Index of the gap bin for a gap between calls. Bins are closed below and +/// open above, so a gap of exactly five minutes lands in `5–10`. +#[must_use] +pub fn cache_gap_bin(gap_seconds: u64) -> usize { + CACHE_GAP_BIN_UPPER_MINUTES + .iter() + .position(|upper| gap_seconds < upper * 60) + .unwrap_or(CACHE_GAP_BIN_COUNT - 1) +} + +/// Label of a gap bin, such as `<1m`, `5–10m`, or `60m+`. +#[must_use] +pub fn cache_gap_bin_label(index: usize) -> String { + let lower = index + .checked_sub(1) + .and_then(|previous| CACHE_GAP_BIN_UPPER_MINUTES.get(previous)); + match (lower, CACHE_GAP_BIN_UPPER_MINUTES.get(index)) { + (None, Some(upper)) => format!("<{upper}m"), + (Some(lower), Some(upper)) => format!("{lower}–{upper}m"), + (Some(lower), None) => format!("{lower}m+"), + (None, None) => String::new(), + } +} + +/// Cached reads as a share of all logical input: ordinary input, cache writes, +/// and cache reads. Token-weighted, so a long call counts for its size. +#[must_use] +pub fn cache_hit_ratio( + input_tokens: u64, + cache_creation_tokens: u64, + cache_read_tokens: u64, +) -> Option { + let logical = input_tokens + .saturating_add(cache_creation_tokens) + .saturating_add(cache_read_tokens); + (logical > 0).then(|| cache_read_tokens as f64 / logical as f64) +} diff --git a/crates/statsai-core/src/cache/report.rs b/crates/statsai-core/src/cache/report.rs new file mode 100644 index 0000000..dce04bb --- /dev/null +++ b/crates/statsai-core/src/cache/report.rs @@ -0,0 +1,346 @@ +use super::{ + cache_gap_bin_label, cache_hit_ratio, provider_has_cache_diagnostics, AnalyzedCacheCall, + CacheBoundaryCounts, CacheCallClass, CacheHealthV1, CACHE_GAP_BIN_UPPER_MINUTES, + CACHE_HEALTH_VERSION, +}; +use crate::types::{UsageCounts, UsageEvent, UsageSummary}; +use chrono::{DateTime, Duration, Offset, TimeZone, Utc}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +pub const CACHE_REPORT_SCHEMA_VERSION: &str = "cache_report.v1"; + +/// Width of a `--timeline` bucket. +pub const CACHE_TIMELINE_BUCKET_MINUTES: i64 = 10; + +/// The prompt-cache report for a period, as `statsai report cache --json` and +/// the local API print it. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct CacheReport { + pub schema_version: String, + pub detector_version: u32, + pub label: String, + pub since: Option>, + pub until: DateTime, + pub filters: CacheReportFilters, + /// Token totals across every provider in scope. + pub totals: CacheTokenTotals, + /// Detector counts across the providers it covers. + pub diagnostics: CacheHealthV1, + pub boundaries: CacheBoundaryCounts, + pub gap_histogram: Vec, + pub providers: Vec, + /// UTC days, matching the daily summaries that sync. + pub days: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timeline: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub details: Option>, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct CacheReportFilters { + pub provider: Option, + pub account: Option, + pub session: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct CacheTokenTotals { + pub requests: u64, + pub input_tokens: u64, + pub cache_creation_tokens: u64, + pub cache_read_tokens: u64, + /// Ordinary input, cache writes, and cache reads: the context processed. + pub logical_input_tokens: u64, + /// Cache reads over logical input. + pub cache_hit_ratio: Option, + pub avg_logical_input_per_request: Option, +} + +impl CacheTokenTotals { + pub fn add_event(&mut self, event: &UsageEvent) { + self.add_usage(&event.usage, 1); + } + + /// Adds a summary that has no events behind it, such as a provider's own + /// session totals. It counts toward token totals, never diagnostics. + pub fn add_summary(&mut self, summary: &UsageSummary) { + self.add_usage(&summary.usage, 0); + } + + fn add_usage(&mut self, usage: &UsageCounts, default_requests: u64) { + self.requests = self + .requests + .saturating_add(usage.requests.unwrap_or(default_requests)); + self.input_tokens = self + .input_tokens + .saturating_add(usage.input_tokens.unwrap_or(0)); + self.cache_creation_tokens = self + .cache_creation_tokens + .saturating_add(usage.cache_creation_tokens.unwrap_or(0)); + self.cache_read_tokens = self + .cache_read_tokens + .saturating_add(usage.cache_read_tokens.unwrap_or(0)); + self.finish(); + } + + fn finish(&mut self) { + self.logical_input_tokens = self + .input_tokens + .saturating_add(self.cache_creation_tokens) + .saturating_add(self.cache_read_tokens); + self.cache_hit_ratio = cache_hit_ratio( + self.input_tokens, + self.cache_creation_tokens, + self.cache_read_tokens, + ); + self.avg_logical_input_per_request = + (self.requests > 0).then(|| self.logical_input_tokens as f64 / self.requests as f64); + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct CacheGapRow { + pub label: String, + pub min_minutes: u64, + /// Exclusive upper bound; `None` for the open last bin. + pub max_minutes: Option, + pub comparable: u64, + pub losses: u64, + /// Reusable context the bin's suspected losses did not read from the cache. + pub missed_tokens: u64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct CacheProviderRow { + pub provider: String, + pub totals: CacheTokenTotals, + /// `None` for providers the detector does not cover. + pub diagnostics: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct CacheDayRow { + pub day: String, + pub totals: CacheTokenTotals, + pub diagnostics: Option, +} + +/// One local-time bucket of model calls, keyed by when each call completed. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct CacheTimelineRow { + /// Bucket start in local time, RFC 3339 with offset. + pub start: String, + pub requests: u64, + pub logical_input_tokens: u64, + pub cache_read_tokens: u64, + pub cache_hit_ratio: Option, + pub first_observed: u64, + pub boundaries: u64, + pub losses: u64, + pub missed_tokens: u64, +} + +/// Inputs for [`build_cache_report`]. `events` are the events in scope; +/// `calls` are their analyzed calls, judged against whole sessions. +/// `summaries` are single-day summaries in scope with no events behind them, +/// which sync with the daily summaries and count toward the same totals. +pub struct CacheReportInput<'a> { + pub label: String, + pub since: Option>, + pub until: DateTime, + pub filters: CacheReportFilters, + pub events: &'a [UsageEvent], + pub summaries: &'a [UsageSummary], + pub calls: Vec, + pub include_details: bool, +} + +#[must_use] +pub fn build_cache_report( + input: CacheReportInput<'_>, + timeline: Option<&Tz>, +) -> CacheReport +where + Tz::Offset: std::fmt::Display, +{ + let mut totals = CacheTokenTotals::default(); + let mut diagnostics = CacheHealthV1::default(); + let mut boundaries = CacheBoundaryCounts::default(); + let mut providers: BTreeMap)> = + BTreeMap::new(); + let mut days: BTreeMap)> = BTreeMap::new(); + let mut event_days: BTreeMap<&str, String> = BTreeMap::new(); + + for event in input.events { + let supported = provider_has_cache_diagnostics(&event.provider); + let day = event.session.started_at.date_naive().to_string(); + event_days.insert(event.event_id.0.as_str(), day.clone()); + totals.add_event(event); + for (totals, health) in [ + providers.entry(event.provider.clone()).or_default(), + days.entry(day).or_default(), + ] { + totals.add_event(event); + if supported { + health + .get_or_insert_with(CacheHealthV1::default) + .add_event_input(event); + } + } + if supported { + diagnostics.add_event_input(event); + } + } + for summary in input.summaries { + let day = summary + .period_start + .unwrap_or(summary.observed_at) + .date_naive() + .to_string(); + totals.add_summary(summary); + providers + .entry(summary.provider.clone()) + .or_default() + .0 + .add_summary(summary); + days.entry(day).or_default().0.add_summary(summary); + } + for call in &input.calls { + diagnostics.add_call(call); + boundaries.add_call(call); + if let Some((_, health)) = providers.get_mut(&call.provider) { + health + .get_or_insert_with(CacheHealthV1::default) + .add_call(call); + } + if let Some((_, health)) = event_days + .get(call.event_id.0.as_str()) + .and_then(|day| days.get_mut(day)) + { + health + .get_or_insert_with(CacheHealthV1::default) + .add_call(call); + } + } + + CacheReport { + schema_version: CACHE_REPORT_SCHEMA_VERSION.to_string(), + detector_version: CACHE_HEALTH_VERSION, + label: input.label, + since: input.since, + until: input.until, + filters: input.filters, + totals, + gap_histogram: gap_histogram(&diagnostics), + diagnostics, + boundaries, + providers: providers + .into_iter() + .map(|(provider, (totals, diagnostics))| CacheProviderRow { + provider, + totals, + diagnostics, + }) + .collect(), + days: days + .into_iter() + .map(|(day, (totals, diagnostics))| CacheDayRow { + day, + totals, + diagnostics, + }) + .collect(), + timeline: timeline.map(|tz| cache_timeline(&input.calls, tz)), + details: input.include_details.then_some(input.calls), + } +} + +#[must_use] +pub fn gap_histogram(health: &CacheHealthV1) -> Vec { + health + .gap_comparable + .iter() + .zip(&health.gap_losses) + .zip(&health.gap_missed_tokens) + .enumerate() + .map( + |(index, ((comparable, losses), missed_tokens))| CacheGapRow { + label: cache_gap_bin_label(index), + min_minutes: index + .checked_sub(1) + .and_then(|previous| CACHE_GAP_BIN_UPPER_MINUTES.get(previous)) + .copied() + .unwrap_or(0), + max_minutes: CACHE_GAP_BIN_UPPER_MINUTES.get(index).copied(), + comparable: *comparable, + losses: *losses, + missed_tokens: *missed_tokens, + }, + ) + .collect() +} + +/// Groups calls into [`CACHE_TIMELINE_BUCKET_MINUTES`] buckets of local time. +/// Buckets without calls are left out. +#[must_use] +pub fn cache_timeline(calls: &[AnalyzedCacheCall], tz: &Tz) -> Vec +where + Tz::Offset: std::fmt::Display, +{ + let width = Duration::minutes(CACHE_TIMELINE_BUCKET_MINUTES).num_seconds(); + let mut buckets: BTreeMap, CacheTimelineRow> = BTreeMap::new(); + for call in calls { + // Truncating local wall-clock seconds keeps buckets on local ten-minute + // marks in zones whose offset is not a whole number of hours. + let offset = i64::from( + call.completed_at + .with_timezone(tz) + .offset() + .fix() + .local_minus_utc(), + ); + let local_seconds = call.completed_at.timestamp() + offset; + let start_seconds = local_seconds - local_seconds.rem_euclid(width) - offset; + let Some(start) = DateTime::::from_timestamp(start_seconds, 0) else { + continue; + }; + let row = buckets.entry(start).or_insert_with(|| CacheTimelineRow { + start: start.with_timezone(tz).to_rfc3339(), + requests: 0, + logical_input_tokens: 0, + cache_read_tokens: 0, + cache_hit_ratio: None, + first_observed: 0, + boundaries: 0, + losses: 0, + missed_tokens: 0, + }); + row.requests = row.requests.saturating_add(call.requests); + row.logical_input_tokens = row + .logical_input_tokens + .saturating_add(call.logical_input_tokens()); + row.cache_read_tokens = row.cache_read_tokens.saturating_add(call.cache_read_tokens); + match call.verdict.class { + CacheCallClass::FirstObserved => row.first_observed += 1, + CacheCallClass::Boundary => row.boundaries += 1, + _ => {} + } + if call.verdict.loss.is_some() { + row.losses += 1; + } + row.missed_tokens = row + .missed_tokens + .saturating_add(call.verdict.missed_tokens.unwrap_or(0)); + } + buckets + .into_values() + .map(|mut row| { + row.cache_hit_ratio = (row.logical_input_tokens > 0) + .then(|| row.cache_read_tokens as f64 / row.logical_input_tokens as f64); + row + }) + .collect() +} diff --git a/crates/statsai-core/src/lib.rs b/crates/statsai-core/src/lib.rs index 6292f72..8f2e9eb 100644 --- a/crates/statsai-core/src/lib.rs +++ b/crates/statsai-core/src/lib.rs @@ -2,6 +2,7 @@ mod account_plan; mod archive; +mod cache; mod code_changes; mod daemon_presence; mod ids; @@ -14,6 +15,7 @@ mod types; pub use account_plan::*; pub use archive::*; +pub use cache::*; pub use code_changes::*; pub use daemon_presence::*; pub use ids::*; diff --git a/crates/statsai-core/src/tests/cache.rs b/crates/statsai-core/src/tests/cache.rs new file mode 100644 index 0000000..6fd1230 --- /dev/null +++ b/crates/statsai-core/src/tests/cache.rs @@ -0,0 +1,849 @@ +use super::support::{test_event, test_source}; +use super::*; +use chrono::{DateTime, Duration, FixedOffset, TimeZone, Utc}; + +fn at(seconds: i64) -> DateTime { + Utc.with_ymd_and_hms(2026, 3, 2, 10, 0, 0).unwrap() + Duration::seconds(seconds) +} + +/// One single-call event in session `s`, transcript `main`, at `seconds`. +fn call(index: u64, seconds: i64, input: u64, read: u64, write: Option) -> UsageEvent { + let source = test_source("claude_code", "/fixture/claude"); + let mut event = test_event("claude_code", &source, at(seconds), 0, None); + event.event_id = EventId(format!("event_{index:03}")); + event.model = Some(ModelInfo { + normalized_name: Some("claude-model".to_string()), + ..ModelInfo::default() + }); + event.usage = UsageCounts { + input_tokens: Some(input), + cache_read_tokens: Some(read), + cache_creation_tokens: write, + output_tokens: Some(10), + requests: Some(1), + ..UsageCounts::default() + }; + event.parse_evidence = Some(ParseEvidence { + event_key_version: "test".to_string(), + source_file_path_hash: Some("main".to_string()), + source_line_number: Some(index), + source_record_id: None, + model_inferred: false, + timestamp_inferred: false, + account_identity_source: IdentitySource::Unresolved, + }); + event +} + +fn classes(calls: &[AnalyzedCacheCall]) -> Vec { + calls.iter().map(|call| call.verdict.class).collect() +} + +fn losses(calls: &[AnalyzedCacheCall]) -> Vec> { + calls.iter().map(|call| call.verdict.loss).collect() +} + +fn health(calls: &[AnalyzedCacheCall]) -> CacheHealthV1 { + let mut health = CacheHealthV1::default(); + for call in calls { + health.add_call(call); + } + health +} + +#[test] +fn gap_bins_are_closed_below_and_open_above() { + let cases = [ + (0, 0), + (59, 0), + (60, 1), + (299, 1), + (300, 2), + (599, 2), + (600, 3), + (899, 3), + (900, 4), + (1_799, 4), + (1_800, 5), + (2_699, 5), + (2_700, 6), + (3_599, 6), + (3_600, 7), + (86_400, 7), + ]; + for (seconds, bin) in cases { + assert_eq!(cache_gap_bin(seconds), bin, "{seconds}s"); + } + let labels = (0..CACHE_GAP_BIN_COUNT) + .map(cache_gap_bin_label) + .collect::>(); + assert_eq!( + labels, + ["<1m", "1–5m", "5–10m", "10–15m", "15–30m", "30–45m", "45–60m", "60m+"] + ); +} + +#[test] +fn hit_ratio_counts_cache_writes_in_the_denominator() { + assert_eq!(cache_hit_ratio(100, 300, 600), Some(0.6)); + assert_eq!(cache_hit_ratio(0, 0, 0), None); +} + +#[test] +fn growing_context_is_reuse_not_loss() { + let events = [ + call(1, 0, 10, 0, Some(20_000)), + call(2, 30, 10, 20_000, Some(2_000)), + call(3, 60, 10, 22_000, Some(3_000)), + call(4, 90, 10, 25_000, Some(500)), + ]; + let calls = analyze_cache_calls(&events); + assert_eq!( + classes(&calls), + [ + CacheCallClass::FirstObserved, + CacheCallClass::Comparable, + CacheCallClass::Comparable, + CacheCallClass::Comparable + ] + ); + assert!(losses(&calls).iter().all(Option::is_none)); + assert_eq!(calls[2].verdict.baseline_tokens, Some(22_000)); + let health = health(&calls); + assert_eq!((health.first_observed, health.first_cold), (1, 1)); + assert_eq!(health.gap_comparable[0], 3); +} + +#[test] +fn losses_need_half_the_baseline_and_the_token_floor() { + // Each pair: a call leaving 10,000 reusable tokens, then the probe, which + // writes to the cache whatever it did not read. + let probe = |read: u64| { + let events = [ + call(1, 0, 0, 10_000, Some(0)), + call(2, 10, 0, read, Some(20_000 - read)), + ]; + analyze_cache_calls(&events)[1].verdict.loss + }; + assert_eq!(probe(0), Some(CacheLoss::Full)); + assert_eq!(probe(5_000), Some(CacheLoss::Partial)); + assert_eq!(probe(5_001), None, "a drop under half is not a loss"); + + // A drop of more than half that is still under the token floor. + let events = [ + call(1, 0, 0, 8_000, Some(0)), + call(2, 10, 8_000, 3_905, Some(0)), + ]; + assert_eq!(analyze_cache_calls(&events)[1].verdict.loss, None); + let events = [ + call(1, 0, 0, 8_192, Some(0)), + call(2, 10, 8_192, 4_096, Some(0)), + ]; + assert_eq!( + analyze_cache_calls(&events)[1].verdict.loss, + Some(CacheLoss::Partial) + ); +} + +#[test] +fn the_baseline_is_bounded_by_the_current_input() { + let events = [ + call(1, 0, 0, 100_000, Some(0)), + // The context shrank to 20,000 tokens; 15,000 of them were reused. + call(2, 10, 5_000, 15_000, Some(0)), + ]; + let verdict = &analyze_cache_calls(&events)[1].verdict; + assert_eq!(verdict.baseline_tokens, Some(20_000)); + assert_eq!(verdict.loss, None); +} + +#[test] +fn without_write_telemetry_previous_reads_are_the_baseline() { + let events = [ + call(1, 0, 40_000, 50_000, None), + call(2, 10, 45_000, 50_000, None), + ]; + let calls = analyze_cache_calls(&events); + assert_eq!(calls[1].verdict.baseline_tokens, Some(50_000)); + assert_eq!(calls[1].verdict.loss, None); +} + +#[test] +fn a_tiny_previous_context_is_too_small_to_judge() { + let events = [ + call(1, 0, 100, 0, Some(3_000)), + call(2, 10, 100, 0, Some(3_000)), + ]; + let calls = analyze_cache_calls(&events); + assert_eq!(calls[1].verdict.class, CacheCallClass::BelowThreshold); + assert_eq!(health(&calls).below_threshold, 1); +} + +#[test] +fn sub_agents_are_separate_streams() { + let mut events = vec![ + call(1, 0, 0, 100_000, Some(1_000)), + call(2, 20, 0, 0, Some(20_000)), + call(3, 40, 0, 101_000, Some(1_000)), + call(4, 60, 0, 20_000, Some(500)), + ]; + // In one stream, the small agent context reads as a drop. + assert!(analyze_cache_calls(&events) + .iter() + .any(|call| call.verdict.loss.is_some())); + + for index in [1, 3] { + events[index].context = Some(CallContext { + agent_hash: Some("agent".to_string()), + ..CallContext::default() + }); + } + let calls = analyze_cache_calls(&events); + assert!(losses(&calls).iter().all(Option::is_none)); + assert_eq!(health(&calls).first_observed, 2); + + // A different transcript is a different stream too. + let mut events = vec![ + call(1, 0, 0, 100_000, Some(1_000)), + call(2, 20, 0, 5_000, Some(500)), + ]; + if let Some(evidence) = events[1].parse_evidence.as_mut() { + evidence.source_file_path_hash = Some("agent-file".to_string()); + } + assert_eq!(health(&analyze_cache_calls(&events)).first_observed, 2); +} + +#[test] +fn model_changes_and_compaction_restart_the_comparison() { + let mut events = vec![ + call(1, 0, 0, 100_000, Some(0)), + call(2, 10, 0, 0, Some(100_000)), + call(3, 20, 0, 100_000, Some(0)), + call(4, 30, 0, 0, Some(30_000)), + ]; + events[1].model = Some(ModelInfo { + normalized_name: Some("other-model".to_string()), + ..ModelInfo::default() + }); + events[2].model = events[1].model.clone(); + events[3].model = events[1].model.clone(); + events[3].context = Some(CallContext { + after_compaction: true, + ..CallContext::default() + }); + let calls = analyze_cache_calls(&events); + assert_eq!( + calls[1].verdict.boundary, + Some(CacheBoundaryReason::ModelChange) + ); + assert_eq!( + calls[1].verdict.previous_model.as_deref(), + Some("claude-model") + ); + assert_eq!(calls[2].verdict.class, CacheCallClass::Comparable); + assert_eq!( + calls[3].verdict.boundary, + Some(CacheBoundaryReason::Compaction) + ); + assert!(losses(&calls).iter().all(Option::is_none)); + assert_eq!(health(&calls).boundaries, 2); +} + +#[test] +fn a_call_without_a_timestamp_is_counted_but_not_placed() { + let mut events = vec![ + call(1, 0, 0, 50_000, Some(0)), + call(2, 10, 0, 0, Some(0)), + call(3, 20, 0, 50_000, Some(0)), + ]; + if let Some(evidence) = events[1].parse_evidence.as_mut() { + evidence.timestamp_inferred = true; + } + let calls = analyze_cache_calls(&events); + let unplaced = calls + .iter() + .find(|call| call.event_id.0 == "event_002") + .expect("unplaced call"); + assert_eq!( + unplaced.verdict.unclassifiable, + Some(CacheUnclassifiableReason::TimestampMissing) + ); + let third = calls + .iter() + .find(|call| call.event_id.0 == "event_003") + .expect("third call"); + assert_eq!(third.verdict.class, CacheCallClass::Comparable); + assert_eq!(third.verdict.loss, None); +} + +#[test] +fn calls_with_one_timestamp_and_no_recorded_order_are_unclassifiable() { + // The cold call sorts first by event id, but either could have come first, + // so neither is judged against the call before them. + let mut events = vec![ + call(1, 0, 0, 50_000, Some(0)), + call(2, 10, 0, 0, Some(0)), + call(3, 10, 0, 50_000, Some(0)), + call(4, 20, 0, 50_000, Some(0)), + ]; + for event in &mut events[1..3] { + if let Some(evidence) = event.parse_evidence.as_mut() { + evidence.source_line_number = None; + } + } + let calls = analyze_cache_calls(&events); + for tied in &calls[1..3] { + assert_eq!( + tied.verdict.unclassifiable, + Some(CacheUnclassifiableReason::AmbiguousOrder) + ); + } + assert_eq!( + calls[3].verdict.boundary, + Some(CacheBoundaryReason::EvidenceGap) + ); + assert!(losses(&calls).iter().all(Option::is_none)); + + // The same timestamps keep their order when the lines say it. + let events = [ + call(1, 0, 0, 50_000, Some(0)), + call(2, 10, 0, 50_000, Some(0)), + call(3, 10, 0, 0, Some(50_000)), + ]; + assert_eq!( + analyze_cache_calls(&events)[2].verdict.loss, + Some(CacheLoss::Full) + ); +} + +#[test] +fn aggregated_usage_is_unclassifiable_and_breaks_the_chain() { + let mut events = vec![ + call(1, 0, 0, 80_000, Some(0)), + call(2, 60, 3_000, 240_000, Some(0)), + call(3, 120, 0, 0, Some(80_000)), + ]; + events[1].usage.requests = Some(3); + let calls = analyze_cache_calls(&events); + assert_eq!( + calls[1].verdict.unclassifiable, + Some(CacheUnclassifiableReason::Aggregated) + ); + assert_eq!(calls[1].requests, 3); + assert_eq!( + calls[2].verdict.boundary, + Some(CacheBoundaryReason::EvidenceGap) + ); + let health = health(&calls); + assert_eq!( + (health.requests, health.unclassifiable, health.analyzed()), + (5, 3, 2) + ); +} + +#[test] +fn recorded_calls_of_a_turn_are_judged_one_by_one() { + let mut turn = call(1, 0, 0, 0, Some(0)); + turn.provider = "codex".to_string(); + turn.usage.requests = Some(3); + turn.context = Some(CallContext { + calls: vec![ + ModelCall { + completed_at: at(10), + requested_at: Some(at(0)), + input_tokens: Some(30_000), + cache_read_tokens: Some(0), + cache_creation_tokens: Some(0), + after_compaction: false, + model: None, + }, + ModelCall { + completed_at: at(20), + requested_at: Some(at(12)), + input_tokens: Some(2_000), + cache_read_tokens: Some(29_952), + cache_creation_tokens: Some(0), + after_compaction: false, + model: None, + }, + ModelCall { + completed_at: at(400), + requested_at: Some(at(390)), + input_tokens: Some(31_000), + cache_read_tokens: Some(1_024), + cache_creation_tokens: Some(0), + after_compaction: false, + model: None, + }, + ], + ..CallContext::default() + }); + let calls = analyze_cache_calls([&turn]); + assert_eq!(calls.len(), 3); + assert_eq!(calls[1].verdict.loss, None); + assert_eq!(calls[2].verdict.loss, Some(CacheLoss::Partial)); + assert_eq!(calls[2].verdict.gap_seconds, Some(370)); + assert_eq!( + calls[2].verdict.gap_timing, + Some(CacheGapTiming::RequestStart) + ); + + // A breakdown that does not cover every call is not used. + if let Some(context) = turn.context.as_mut() { + context.calls.pop(); + } + let calls = analyze_cache_calls([&turn]); + assert_eq!( + calls[0].verdict.unclassifiable, + Some(CacheUnclassifiableReason::Aggregated) + ); +} + +#[test] +fn a_model_change_inside_a_turn_is_a_boundary_not_a_loss() { + let mut turn = call(1, 0, 0, 0, Some(0)); + turn.provider = "codex".to_string(); + turn.usage.requests = Some(2); + let model_call = |seconds, read, write, model: &str| ModelCall { + completed_at: at(seconds), + requested_at: None, + input_tokens: Some(1_000), + cache_read_tokens: Some(read), + cache_creation_tokens: Some(write), + after_compaction: false, + model: Some(model.to_string()), + }; + turn.context = Some(CallContext { + calls: vec![ + model_call(10, 50_000, 0, "gpt-5"), + model_call(20, 0, 0, "gpt-5.1"), + ], + ..CallContext::default() + }); + let calls = analyze_cache_calls([&turn]); + assert_eq!( + calls[1].verdict.boundary, + Some(CacheBoundaryReason::ModelChange) + ); + assert_eq!(calls[1].verdict.previous_model.as_deref(), Some("gpt-5")); + assert_eq!(calls[1].verdict.loss, None); +} + +#[test] +fn a_turn_call_without_input_counts_is_unclassifiable() { + let mut turn = call(1, 0, 0, 0, Some(0)); + turn.provider = "codex".to_string(); + turn.usage.requests = Some(3); + let model_call = |seconds, input: Option, read: Option| ModelCall { + completed_at: at(seconds), + requested_at: None, + input_tokens: input, + cache_read_tokens: read, + cache_creation_tokens: None, + after_compaction: false, + model: None, + }; + turn.context = Some(CallContext { + calls: vec![ + model_call(10, Some(1_000), Some(50_000)), + model_call(20, None, None), + model_call(30, Some(1_000), Some(0)), + ], + ..CallContext::default() + }); + let calls = analyze_cache_calls([&turn]); + assert_eq!( + calls[1].verdict.unclassifiable, + Some(CacheUnclassifiableReason::NoInputTelemetry) + ); + // The call after it has no known predecessor, so it is not a loss. + assert_eq!( + calls[2].verdict.boundary, + Some(CacheBoundaryReason::EvidenceGap) + ); + assert_eq!(calls[2].verdict.loss, None); +} + +#[test] +fn gaps_prefer_the_recorded_request_start() { + let mut events = vec![ + call(1, 0, 0, 50_000, Some(0)), + call(2, 400, 0, 50_000, Some(0)), + call(3, 800, 0, 50_000, Some(0)), + ]; + events[1].context = Some(CallContext { + requested_at: Some(at(330)), + ..CallContext::default() + }); + // A request start before the previous response belongs to another exchange. + events[2].context = Some(CallContext { + requested_at: Some(at(390)), + ..CallContext::default() + }); + let calls = analyze_cache_calls(&events); + assert_eq!(calls[1].verdict.gap_seconds, Some(330)); + assert_eq!( + calls[1].verdict.gap_timing, + Some(CacheGapTiming::RequestStart) + ); + assert_eq!(calls[2].verdict.gap_seconds, Some(400)); + assert_eq!( + calls[2].verdict.gap_timing, + Some(CacheGapTiming::RecordedEvents) + ); + let health = health(&calls); + assert_eq!(health.timing_estimated, 1); + assert_eq!(health.gap_comparable[2], 2); +} + +#[test] +fn a_short_gap_loss_is_counted_in_its_bin() { + let events = [ + call(1, 0, 2, 350_000, Some(1_000)), + call(2, 40, 2, 27_000, Some(325_000)), + call(3, 80, 2, 352_000, Some(2_000)), + ]; + let calls = analyze_cache_calls(&events); + assert_eq!(losses(&calls), [None, Some(CacheLoss::Partial), None]); + let health = health(&calls); + assert_eq!((health.gap_comparable[0], health.gap_losses[0]), (2, 1)); + assert_eq!((health.losses, health.full_losses), (1, 0)); +} + +#[test] +fn a_loss_after_a_long_pause_is_counted_in_the_open_bin() { + let events = [ + call(1, 0, 2, 120_000, Some(1_000)), + call(2, 2 * 3_600, 2, 0, Some(121_500)), + ]; + let calls = analyze_cache_calls(&events); + assert_eq!(calls[1].verdict.loss, Some(CacheLoss::Full)); + let health = health(&calls); + assert_eq!((health.gap_comparable[7], health.gap_losses[7]), (1, 1)); +} + +#[test] +fn previous_write_lifetimes_are_kept_as_evidence() { + let mut events = vec![ + call(1, 0, 2, 10_000, Some(9_000)), + call(2, 600, 2, 0, Some(19_000)), + ]; + events[0].usage.cache_creation_5m_tokens = Some(4_000); + events[0].usage.cache_creation_1h_tokens = Some(5_000); + let calls = analyze_cache_calls(&events); + assert_eq!(calls[0].write_ttl, Some(CacheWriteTtl::Mixed)); + assert_eq!( + calls[1].verdict.previous_write_ttl, + Some(CacheWriteTtl::Mixed) + ); +} + +#[test] +fn health_counts_add_up_and_merge() { + let mut events = vec![ + call(1, 0, 0, 0, Some(40_000)), + call(2, 30, 0, 40_000, Some(1_000)), + call(3, 400, 41_000, 0, Some(0)), + call(4, 460, 0, 41_000, Some(0)), + call(5, 520, 9_000, 0, Some(0)), + ]; + events[4].usage.requests = Some(2); + let calls = analyze_cache_calls(&events); + let health = health(&calls); + assert_eq!(health.uncached, 1, "the third call skipped the cache"); + assert_eq!( + health.requests, + health.unclassifiable + + health.first_observed + + health.boundaries + + health.below_threshold + + health.uncached + + health.comparable + ); + assert_eq!(health.gap_comparable.iter().sum::(), health.comparable); + assert_eq!(health.gap_losses.iter().sum::(), health.losses); + assert_eq!( + health.gap_missed_tokens.iter().sum::(), + health.missed_tokens + ); + + let mut merged = CacheHealthV1::default(); + merged.merge(&health); + merged.merge(&health); + assert_eq!(merged.requests, health.requests * 2); + assert_eq!(merged.gap_comparable[2], health.gap_comparable[2] * 2); +} + +#[test] +fn providers_without_call_order_are_left_out() { + let mut event = call(1, 0, 0, 1_000, Some(0)); + event.provider = "cursor".to_string(); + assert!(analyze_cache_calls([&event]).is_empty()); +} + +#[test] +fn the_analysis_does_not_depend_on_input_order() { + let events = vec![ + call(1, 0, 0, 0, Some(30_000)), + call(2, 20, 0, 30_000, Some(0)), + call(3, 700, 30_000, 0, Some(0)), + ]; + let mut reversed = events.clone(); + reversed.reverse(); + assert_eq!(analyze_cache_calls(&events), analyze_cache_calls(&reversed)); +} + +#[test] +fn the_report_splits_totals_by_provider_and_utc_day() { + let mut events = vec![ + call(1, 13 * 3_600 + 59 * 60, 100, 0, Some(20_000)), + call(2, 14 * 3_600 + 1, 100, 20_000, Some(100)), + ]; + let mut cursor = call(3, 0, 1_000, 3_000, None); + cursor.provider = "cursor".to_string(); + events.push(cursor); + let calls = analyze_cache_calls(&events); + let report = build_cache_report( + CacheReportInput { + label: "fixture".to_string(), + since: None, + until: at(86_400), + filters: CacheReportFilters::default(), + events: &events, + summaries: &[], + calls, + include_details: false, + }, + Some(&FixedOffset::east_opt(5 * 3_600 + 30 * 60).unwrap()), + ); + assert_eq!(report.totals.requests, 3); + assert_eq!(report.totals.logical_input_tokens, 44_300); + assert_eq!(report.providers.len(), 2); + assert!(report.providers[1].diagnostics.is_none(), "cursor"); + assert_eq!( + report.diagnostics.write_split_unavailable_input_tokens, 0, + "only covered providers count unsplit input" + ); + assert_eq!( + report + .days + .iter() + .map(|day| day.day.as_str()) + .collect::>(), + ["2026-03-02", "2026-03-03"] + ); + let timeline = report.timeline.expect("timeline"); + assert_eq!(timeline.len(), 2); + assert_eq!(timeline[0].start, "2026-03-03T05:20:00+05:30"); + assert_eq!(report.gap_histogram[0].comparable, 0); + assert_eq!(report.gap_histogram[1].label, "1–5m"); + assert!(report.details.is_none()); +} + +#[test] +fn call_context_is_omitted_when_empty_and_optional_on_read() { + let event = call(1, 0, 10, 20, Some(30)); + let value = serde_json::to_value(&event).expect("event json"); + assert!(value.get("context").is_none()); + let mut legacy = value.clone(); + legacy.as_object_mut().expect("object").remove("context"); + let read: UsageEvent = serde_json::from_value(legacy).expect("legacy event"); + assert_eq!(read.context, None); + + let context = CallContext { + requested_at: Some(at(0)), + ..CallContext::default() + }; + assert_eq!( + serde_json::to_value(&context).expect("context json"), + serde_json::json!({ "requested_at": "2026-03-02T10:00:00Z" }) + ); +} + +#[test] +fn removing_cache_health_restores_the_summary_a_receiver_stores() { + let metrics = |cache_health| SummaryMetrics { + active_seconds: None, + tracked_requests: None, + tracked_output_tokens: None, + tracked_reasoning_tokens: None, + latency_ms: None, + time_to_first_token_ms: None, + generated_tps: None, + visible_tps: None, + overall_generated_tps: None, + overall_visible_tps: None, + cache_hit_ratio: None, + reasoning_share: None, + total_messages: Some(4), + user_messages: None, + assistant_messages: None, + developer_messages: None, + cache_health, + }; + let only_health = metrics(Some(CacheHealthV1::default())); + let mut kept = only_health.clone(); + kept.total_messages = Some(4); + let mut empty = only_health; + empty.total_messages = None; + assert!(!empty.is_empty()); + + let summary = |metrics| UsageSummary { + metrics, + ..serde_json::from_value::(fixture_summary_json()).expect("summary") + }; + assert_eq!( + summary(Some(empty)).without_cache_health().metrics, + None, + "metrics holding only cache_health go away" + ); + assert_eq!( + summary(Some(kept)).without_cache_health().metrics, + Some(metrics(None)) + ); +} + +fn fixture_summary_json() -> serde_json::Value { + serde_json::json!({ + "schema_version": USAGE_SUMMARY_SCHEMA_VERSION, + "summary_id": "summary_fixture", + "device_id": "device", + "provider": "claude_code", + "source_id": "source", + "provider_account_id": null, + "source": { + "adapter_id": "test", + "adapter_version": "0", + "source_kind": "local_adapter", + "location_origin": null, + "source_type": "jsonl", + "source_path_hash": null, + "source_record_id": null, + "parse_confidence": "high" + }, + "model": null, + "usage": {}, + "cost": { "currency": "USD", "estimated_api_equivalent_usd": null, "provider_reported_usd": null, "pricing_source": null, "pricing_version": null, "confidence": "low" }, + "parse_evidence": null, + "project": null, + "privacy": { "mode": "metadata_only", "contains_prompt_text": false, "contains_response_text": false, "contains_file_paths": false }, + "metrics": null, + "period_start": null, + "period_end": null, + "observed_at": "2026-03-02T10:00:00Z", + "metadata": { "summary_format": "daily_rollup.v1", "summary_version": null, "total_sessions": null, "total_messages": null, "last_computed_at": null }, + "imported_at": "2026-03-02T10:00:00Z" + }) +} + +#[test] +fn a_call_that_neither_reads_nor_writes_the_cache_is_uncached_not_a_loss() { + // A router serving the request without the cache, between cached calls. + let events = [ + call(1, 0, 10, 0, Some(50_000)), + call(2, 10, 50_010, 0, Some(0)), + call(3, 20, 51_000, 0, Some(0)), + call(4, 30, 10, 50_000, Some(1_000)), + ]; + let calls = analyze_cache_calls(&events); + assert_eq!( + calls + .iter() + .map(|call| call.verdict.class) + .collect::>(), + [ + CacheCallClass::FirstObserved, + CacheCallClass::Uncached, + CacheCallClass::Uncached, + CacheCallClass::BelowThreshold, + ] + ); + assert!(losses(&calls).iter().all(Option::is_none)); + let health = health(&calls); + assert_eq!((health.uncached, health.losses), (2, 0)); + + // Codex reports zero writes on calls that use the cache, so a cold Codex + // call is still judged. + let mut codex = events.clone(); + for event in &mut codex { + event.provider = "codex".to_string(); + } + assert_eq!( + analyze_cache_calls(&codex)[1].verdict.loss, + Some(CacheLoss::Full) + ); +} + +#[test] +fn a_loss_records_the_reusable_context_it_missed() { + let events = [ + call(1, 0, 10, 0, Some(80_000)), + call(2, 30, 10, 80_000, Some(0)), + call(3, 3_900, 10, 20_000, Some(60_000)), + ]; + let calls = analyze_cache_calls(&events); + assert_eq!(calls[1].verdict.missed_tokens, None); + assert_eq!(calls[2].verdict.loss, Some(CacheLoss::Partial)); + assert_eq!(calls[2].verdict.missed_tokens, Some(60_000)); + + let health = health(&calls); + assert_eq!(health.missed_tokens, 60_000); + assert_eq!(health.gap_missed_tokens[CACHE_GAP_BIN_COUNT - 1], 60_000); + let mut doubled = health.clone(); + doubled.merge(&health); + assert_eq!(doubled.missed_tokens, 120_000); + assert_eq!( + gap_histogram(&doubled)[CACHE_GAP_BIN_COUNT - 1].missed_tokens, + 120_000 + ); +} + +#[test] +fn an_inferred_model_does_not_make_a_model_change() { + let mut events = vec![ + call(1, 0, 0, 30_000, Some(0)), + call(2, 10, 0, 1_000, Some(29_000)), + ]; + // The first model is the parser's fallback, the second one the transcript named. + if let Some(evidence) = events[0].parse_evidence.as_mut() { + evidence.model_inferred = true; + } + events[1].model = Some(ModelInfo { + normalized_name: Some("other-model".to_string()), + ..ModelInfo::default() + }); + let calls = analyze_cache_calls(&events); + assert_eq!(calls[1].verdict.boundary, None); + assert_eq!(calls[1].verdict.loss, Some(CacheLoss::Partial)); + assert_eq!(calls[1].verdict.missed_tokens, Some(29_000)); +} + +#[test] +fn unsplit_input_is_counted_per_call_when_writes_are_partly_reported() { + let mut turn = call(1, 0, 3_000, 0, Some(0)); + turn.provider = "codex".to_string(); + turn.usage.requests = Some(2); + let model_call = |seconds, input, write| ModelCall { + completed_at: at(seconds), + requested_at: None, + input_tokens: Some(input), + cache_read_tokens: Some(0), + cache_creation_tokens: write, + after_compaction: false, + model: None, + }; + // The turn's summed usage says zero writes; one call never reported them. + turn.context = Some(CallContext { + calls: vec![model_call(10, 1_000, Some(0)), model_call(20, 2_000, None)], + ..CallContext::default() + }); + let mut health = CacheHealthV1::default(); + health.add_event_input(&turn); + assert_eq!(health.write_split_unavailable_input_tokens, 2_000); + + // Without a call list the event's own counts decide. + let mut single = call(2, 30, 4_000, 0, None); + single.provider = "codex".to_string(); + health.add_event_input(&single); + assert_eq!(health.write_split_unavailable_input_tokens, 6_000); +} diff --git a/crates/statsai-core/src/tests/mod.rs b/crates/statsai-core/src/tests/mod.rs index b39a837..aa2817b 100644 --- a/crates/statsai-core/src/tests/mod.rs +++ b/crates/statsai-core/src/tests/mod.rs @@ -1,5 +1,6 @@ pub(crate) use super::*; +mod cache; mod report; mod support; mod types; diff --git a/crates/statsai-core/src/tests/support.rs b/crates/statsai-core/src/tests/support.rs index ce65aee..b311389 100644 --- a/crates/statsai-core/src/tests/support.rs +++ b/crates/statsai-core/src/tests/support.rs @@ -74,6 +74,7 @@ pub(super) fn test_event( }, created_at: started_at, imported_at: started_at, + context: None, } } diff --git a/crates/statsai-core/src/types/event.rs b/crates/statsai-core/src/types/event.rs index acc7695..76d1bf0 100644 --- a/crates/statsai-core/src/types/event.rs +++ b/crates/statsai-core/src/types/event.rs @@ -396,6 +396,54 @@ pub struct PrivacyInfo { pub contains_file_paths: bool, } +/// Where a usage event sits in its conversation, for the prompt-cache report. +/// +/// Recorded only by adapters that can see the boundaries, and kept on the +/// device: sync strips it from events and only bounded counts leave in daily +/// summaries. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize, JsonSchema)] +pub struct CallContext { + /// Hashed sub-agent identity. Sub-agents share their parent's session but + /// keep their own context, so each one is a separate cache stream. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub agent_hash: Option, + /// When the request was sent, read from the record that triggered it: a + /// prompt or a tool result. Single-call events only. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub requested_at: Option>, + /// The provider compacted the conversation before this request. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub after_compaction: bool, + /// The model calls inside an event that aggregates several, such as a Codex + /// turn, oldest first. Empty for single-call events. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub calls: Vec, +} + +/// One model call inside an aggregated usage event. Token fields follow the +/// additive `UsageCounts` contract: ordinary input excludes cache reads and +/// writes, and `None` means the provider did not report the count. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct ModelCall { + pub completed_at: DateTime, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub requested_at: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_tokens: Option, + /// `None` when the provider did not report cache writes for this call. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_tokens: Option, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub after_compaction: bool, + /// The model that served this call, as `cache_model_name` names it. + /// `None` when the transcript did not say, so a model change cannot be + /// read from it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model: Option, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct UsageEvent { pub schema_version: String, @@ -417,6 +465,10 @@ pub struct UsageEvent { pub privacy: PrivacyInfo, pub created_at: DateTime, pub imported_at: DateTime, + /// Local only: sync removes it, so it is not part of the sync schema. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(skip)] + pub context: Option, } impl UsageCounts { diff --git a/crates/statsai-core/src/types/summary.rs b/crates/statsai-core/src/types/summary.rs index f7c1f71..96171e7 100644 --- a/crates/statsai-core/src/types/summary.rs +++ b/crates/statsai-core/src/types/summary.rs @@ -2,6 +2,7 @@ use super::event::{ CostInfo, EventSource, MetricStats, ModelInfo, ParseEvidence, PrivacyInfo, ProjectInfo, UsageCounts, }; +use crate::cache::CacheHealthV1; use crate::ids::{ProviderAccountId, SourceId, SummaryId}; use chrono::{DateTime, Utc}; use schemars::JsonSchema; @@ -30,6 +31,48 @@ pub struct SummaryMetrics { pub user_messages: Option, pub assistant_messages: Option, pub developer_messages: Option, + /// Prompt-cache diagnostics, for providers whose events record call order. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_health: Option, +} + +impl SummaryMetrics { + /// Whether no metric is set, in which case a summary carries `None`. + #[must_use] + pub fn is_empty(&self) -> bool { + self.active_seconds.is_none() + && self.tracked_requests.is_none() + && self.tracked_output_tokens.is_none() + && self.tracked_reasoning_tokens.is_none() + && self.latency_ms.is_none() + && self.time_to_first_token_ms.is_none() + && self.generated_tps.is_none() + && self.visible_tps.is_none() + && self.overall_generated_tps.is_none() + && self.overall_visible_tps.is_none() + && self.cache_hit_ratio.is_none() + && self.reasoning_share.is_none() + && self.total_messages.is_none() + && self.user_messages.is_none() + && self.assistant_messages.is_none() + && self.developer_messages.is_none() + && self.cache_health.is_none() + } +} + +impl UsageSummary { + /// The summary as a receiver without `cache_health` support stores it: + /// the object removed, and metrics dropped when nothing else is left. + #[must_use] + pub fn without_cache_health(mut self) -> Self { + if let Some(metrics) = self.metrics.as_mut() { + metrics.cache_health = None; + } + if self.metrics.as_ref().is_some_and(SummaryMetrics::is_empty) { + self.metrics = None; + } + self + } } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] diff --git a/crates/statsai-daemon/Cargo.toml b/crates/statsai-daemon/Cargo.toml index a510eb2..7b700c6 100644 --- a/crates/statsai-daemon/Cargo.toml +++ b/crates/statsai-daemon/Cargo.toml @@ -21,6 +21,7 @@ chunked_transfer = "1" httpdate = "1" log = "0.4" notify = { workspace = true, optional = true } +percent-encoding.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/crates/statsai-daemon/src/lib.rs b/crates/statsai-daemon/src/lib.rs index 4a59f33..f744f8c 100644 --- a/crates/statsai-daemon/src/lib.rs +++ b/crates/statsai-daemon/src/lib.rs @@ -89,6 +89,16 @@ fn handle_request(mut request: Request, store: &Arc>, auth_token: & return respond_json(request, StatusCode(200), &health_payload()); } + let (path, query) = url.split_once('?').unwrap_or((url.as_str(), "")); + if path == "/reports/cache" { + let report_query = match cache_report_query(query, chrono::Utc::now()) { + Ok(report_query) => report_query, + Err(error) => return respond_text(request, StatusCode(400), &format!("{error:#}")), + }; + let report = lock_store(store).cache_report(&report_query, &chrono::Local)?; + return respond_json(request, StatusCode(200), &report); + } + let s = lock_store(store); let payload = match url.as_str() { "/status" => json!({ @@ -115,6 +125,47 @@ fn handle_request(mut request: Request, store: &Arc>, auth_token: & respond_json(request, StatusCode(200), &payload) } +/// Reads `from`, `to`, `provider`, `account`, `session`, and `timeline` from +/// a percent-encoded query. Per-call details are left to the CLI. +fn cache_report_query( + query: &str, + now: chrono::DateTime, +) -> Result { + let mut params = std::collections::BTreeMap::new(); + for pair in query.split('&').filter(|pair| !pair.is_empty()) { + let (key, value) = pair.split_once('=').unwrap_or((pair, "")); + let value = percent_encoding::percent_decode_str(value) + .decode_utf8() + .context("query values must be UTF-8")?; + if params.insert(key, value.into_owned()).is_some() { + bail!("repeated query parameter {key}"); + } + } + let (from, to) = (params.remove("from"), params.remove("to")); + let mut report_query = match params.remove("all").as_deref() { + None | Some("0" | "false") => { + statsai_store::CacheReportQuery::for_range(from.as_deref(), to.as_deref(), now)? + } + Some("1" | "true") if from.is_none() && to.is_none() => { + statsai_store::CacheReportQuery::all_time(now) + } + Some("1" | "true") => bail!("all cannot be combined with from or to"), + Some(other) => bail!("all must be true or false, not {other}"), + }; + report_query.provider = params.remove("provider"); + report_query.account = params.remove("account"); + report_query.session = params.remove("session"); + report_query.timeline = match params.remove("timeline").as_deref() { + None | Some("0" | "false") => false, + Some("1" | "true") => true, + Some(other) => bail!("timeline must be true or false, not {other}"), + }; + if let Some(key) = params.keys().next() { + bail!("unknown query parameter {key}"); + } + Ok(report_query) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct HttpRejection { status: StatusCode, diff --git a/crates/statsai-daemon/src/tests.rs b/crates/statsai-daemon/src/tests.rs index 040055a..dc82459 100644 --- a/crates/statsai-daemon/src/tests.rs +++ b/crates/statsai-daemon/src/tests.rs @@ -910,3 +910,127 @@ fn test_task_verification() -> TaskVerification { updated_at: created_at, } } + +#[test] +fn cache_report_query_reads_percent_encoded_filters() { + let now = Utc.with_ymd_and_hms(2026, 7, 10, 12, 0, 0).unwrap(); + let query = cache_report_query( + "from=2026-07-01&to=2026-07-03T00%3A00%3A00%2B02%3A00&provider=claude_code&account=unassigned&session=session_ab&timeline=true", + now, + ) + .expect("query"); + assert_eq!( + query.since, + Some(Utc.with_ymd_and_hms(2026, 7, 1, 0, 0, 0).unwrap()) + ); + assert_eq!( + query.until, + Utc.with_ymd_and_hms(2026, 7, 2, 22, 0, 0).unwrap() + ); + assert_eq!(query.provider.as_deref(), Some("claude_code")); + assert_eq!(query.account.as_deref(), Some("unassigned")); + assert_eq!(query.session.as_deref(), Some("session_ab")); + assert!(query.timeline); + assert!(!query.details, "per-call details stay with the CLI"); + + let default = cache_report_query("", now).expect("default"); + assert_eq!(default.since, Some(now - chrono::Duration::days(7))); + let all = cache_report_query("all=true", now).expect("all"); + assert_eq!((all.since, all.until), (None, now)); + for rejected in [ + "details=true", + "from=a&from=b", + "timeline=maybe", + "from=not-a-date", + "all=true&from=2026-07-01", + "all=maybe", + ] { + assert!(cache_report_query(rejected, now).is_err(), "{rejected}"); + } +} + +#[test] +fn cache_report_route_requires_the_bearer_token() { + assert_eq!( + validate_http_request( + &Method::Get, + "/reports/cache?from=2026-07-01", + &[], + None, + "secret" + ), + Err(HttpRejection { + status: StatusCode(401), + message: "missing or invalid bearer token", + }) + ); +} + +#[test] +fn the_cache_report_route_serves_the_cli_report() { + let store = Store::in_memory().expect("store"); + let event = |id: &str, at: &str, read: u64, write: u64| -> statsai_core::UsageEvent { + serde_json::from_value(json!({ + "schema_version": "usage_event.v1", + "event_id": id, + "device_id": "device", + "provider": "claude_code", + "source_id": "src_daemon_cache", + "provider_account_id": null, + "subscription_id": null, + "source": { + "adapter_id": "test", "adapter_version": "0", "source_kind": "local_adapter", + "location_origin": null, "source_type": "jsonl", "source_path_hash": null, + "source_record_id": null, "parse_confidence": "high" + }, + "session": { + "session_id": "session_daemon", "local_session_id_hash": null, "title": null, + "started_at": at, "ended_at": null, "duration_seconds": null + }, + "model": null, + "usage": { "input_tokens": 10, "cache_read_tokens": read, "cache_creation_tokens": write, "requests": 1 }, + "runtime": null, + "cost": { "currency": "USD", "estimated_api_equivalent_usd": null, "provider_reported_usd": null, "pricing_source": null, "pricing_version": null, "confidence": "low" }, + "parse_evidence": null, + "project": null, + "git": null, + "privacy": { "mode": "metadata_only", "contains_prompt_text": false, "contains_response_text": false, "contains_file_paths": false }, + "created_at": at, + "imported_at": at + })) + .expect("event") + }; + store + .insert_events(&[ + event("event_a", "2026-07-01T10:00:00Z", 0, 50_000), + event("event_b", "2026-07-01T11:30:00Z", 0, 50_010), + ]) + .expect("insert"); + let expected = { + let query = cache_report_query("from=2026-07-01&to=2026-07-02", Utc::now()).expect("query"); + serde_json::to_value(store.cache_report(&query, &chrono::Local).expect("report")) + .expect("json") + }; + assert_eq!(expected["diagnostics"]["losses"], 1); + + let server = Server::http("127.0.0.1:0").expect("server"); + let address = server.server_addr().to_ip().expect("ip address"); + let store = Arc::new(Mutex::new(store)); + let handle = std::thread::spawn(move || { + let request = server.recv().expect("request"); + handle_request(request, &store, "secret").expect("handled"); + }); + let mut stream = std::net::TcpStream::connect(address).expect("connect"); + std::io::Write::write_all( + &mut stream, + b"GET /reports/cache?from=2026-07-01&to=2026-07-02 HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer secret\r\nConnection: close\r\n\r\n", + ) + .expect("request"); + let mut response = String::new(); + std::io::Read::read_to_string(&mut stream, &mut response).expect("response"); + handle.join().expect("server thread"); + assert!(response.starts_with("HTTP/1.1 200"), "{response}"); + let body = response.split_once("\r\n\r\n").expect("body").1; + let served: serde_json::Value = serde_json::from_str(body).expect("json body"); + assert_eq!(served, expected); +} diff --git a/crates/statsai-sdk/src/lib.rs b/crates/statsai-sdk/src/lib.rs index dd6ad9a..56a704b 100644 --- a/crates/statsai-sdk/src/lib.rs +++ b/crates/statsai-sdk/src/lib.rs @@ -8,6 +8,11 @@ pub use statsai_core as core; pub use statsai_store as store; pub use statsai_sync as sync; +/// The prompt-cache report: `Store::cache_report` with a [`CacheReportQuery`] +/// returns the same [`CacheReport`] that `statsai report cache --json` prints. +pub use statsai_core::{AnalyzedCacheCall, CacheHealthV1, CacheReport}; +pub use statsai_store::CacheReportQuery; + use anyhow::{bail, Result}; use chrono::{DateTime, Utc}; use schemars::JsonSchema; diff --git a/crates/statsai-store/src/account_plan/observations.rs b/crates/statsai-store/src/account_plan/observations.rs index 10dd3ea..21ff675 100644 --- a/crates/statsai-store/src/account_plan/observations.rs +++ b/crates/statsai-store/src/account_plan/observations.rs @@ -579,8 +579,7 @@ impl Store { dirty.extend(self.update_event_payload(event)?); changed += 1; } - self.refresh_sync_rollups_for_keys(&dirty.buckets)?; - self.refresh_session_rollups_for_keys(&dirty.sessions)?; + self.refresh_event_rollups(&dirty)?; Ok(changed) } diff --git a/crates/statsai-store/src/cache.rs b/crates/statsai-store/src/cache.rs new file mode 100644 index 0000000..920409a --- /dev/null +++ b/crates/statsai-store/src/cache.rs @@ -0,0 +1,614 @@ +//! Prompt-cache analysis over stored usage events. +//! +//! The detector compares each call with the one before it in its stream, so +//! it always reads whole sessions: a call on the first day of a report, or the +//! first call in a daily bucket, is judged against its real predecessor even +//! when that predecessor falls outside the range. A resumed session can follow +//! its predecessor by days, so no fixed look-back would do. Sessions are read +//! through `usage_events_session_started_idx`, which also bounds the lookup of +//! the calls after a changed one. + +use super::*; +use statsai_core::{ + analyze_cache_calls, build_cache_report, provider_has_cache_diagnostics, AnalyzedCacheCall, + CacheHealthV1, CacheReport, CacheReportFilters, CacheReportInput, CACHE_DIAGNOSTIC_PROVIDERS, + CACHE_HEALTH_VERSION, +}; +use std::collections::VecDeque; + +/// Records which detector version the stored daily summaries were built with. +const CACHE_HEALTH_ROLLUPS_METADATA_KEY: &str = "sync_rollups.cache_health_version"; + +/// Per-target record of whether an HTTP receiver accepts `cache_health`. +const SYNC_CACHE_HEALTH_METADATA_PREFIX: &str = "sync.cache_health_accepted:"; + +/// Sessions kept analyzed during one rollup refresh. Buckets refresh in day +/// order, so the sessions a bucket shares with the next one are the recent +/// ones; a bound keeps a full rebuild from holding every session at once. +const CACHE_SESSION_MEMO_CAPACITY: usize = 64; + +/// Account filter value for events no provider account was attributed to. +pub const CACHE_REPORT_UNASSIGNED_ACCOUNT: &str = "unassigned"; + +/// What `statsai report cache` and the local API ask for. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CacheReportQuery { + pub label: String, + pub since: Option>, + pub until: DateTime, + pub provider: Option, + /// A provider account id, or [`CACHE_REPORT_UNASSIGNED_ACCOUNT`]. + pub account: Option, + /// A hashed `session_…` id, or a prefix of one. + pub session: Option, + pub timeline: bool, + pub details: bool, +} + +impl CacheReportQuery { + /// Resolves `--from` and `--to` like the other reports. Date-only values + /// are UTC days, matching daily summaries; no bounds means the last seven + /// days. + pub fn for_range(from: Option<&str>, to: Option<&str>, now: DateTime) -> Result { + let period = match (from, to) { + (None, None) => statsai_core::ReportPeriod::LastDays(7), + (from, to) => statsai_core::report_period_from_range(from, to, now)?, + }; + Ok(Self::for_period(&period, now)) + } + + /// Every stored call, through now. + pub fn all_time(now: DateTime) -> Self { + Self::for_period(&statsai_core::ReportPeriod::AllTime, now) + } + + fn for_period(period: &statsai_core::ReportPeriod, now: DateTime) -> Self { + let (since, until) = period.published_window(now); + Self { + label: period.label(now), + since, + until, + provider: None, + account: None, + session: None, + timeline: false, + details: false, + } + } +} + +/// A session's calls, by the event that holds them. +type AnalyzedSession = HashMap>; + +/// The columns that place an event in its daily bucket, read without decoding +/// the payload. [`cache_bucket_key`] turns a row of them into the key. +const CACHE_BUCKET_COLUMNS_SQL: &str = "provider, source_id, provider_account_id, started_at, \ + CASE WHEN json_valid(payload) THEN json_extract(payload, '$.project') END"; + +/// Sessions analyzed during one rollup refresh, so the buckets a session spans +/// do not each re-read and re-classify it. +#[derive(Default)] +pub(crate) struct CacheSessionMemo { + sessions: HashMap, + order: VecDeque, +} + +impl Store { + pub fn cache_report( + &self, + query: &CacheReportQuery, + timeline_zone: &Tz, + ) -> Result + where + Tz::Offset: std::fmt::Display, + { + let mut events = Vec::new(); + let mut calls = Vec::new(); + for session_id in self.cache_report_sessions(query)? { + let mut session_events = Vec::new(); + let mut in_scope = BTreeSet::new(); + for (event, in_period) in self.cache_session_events_in_period(&session_id, query)? { + // Another provider's events can share the session id; they + // are counted with the providers that have no diagnostics. + if !provider_has_cache_diagnostics(&event.provider) { + continue; + } + if in_period && cache_report_includes(query, &event) { + in_scope.insert(event.event_id.0.clone()); + events.push(event.clone()); + } + session_events.push(event); + } + calls.extend( + analyze_and_price(&session_events) + .into_iter() + .filter(|call| in_scope.contains(&call.event_id.0)), + ); + } + // Other providers have no diagnostics but count toward the token totals. + events.extend( + self.cache_report_other_events(query)? + .into_iter() + .filter(|event| cache_report_includes(query, event)), + ); + events.sort_by(|left, right| { + left.session + .started_at + .cmp(&right.session.started_at) + .then_with(|| left.event_id.0.cmp(&right.event_id.0)) + }); + // Summaries with no events behind them sync alongside the daily + // rollups, so they count toward the same token totals. A session + // filter has no summaries to match. + let summaries = if query.session.is_some() { + Vec::new() + } else { + self.summaries()? + .into_iter() + .filter(|summary| { + is_http_rollup_passthrough_summary(summary) + && cache_report_includes_summary(query, summary) + }) + .collect::>() + }; + calls.sort_by(|left, right| { + left.completed_at + .cmp(&right.completed_at) + .then_with(|| left.event_id.0.cmp(&right.event_id.0)) + .then_with(|| left.call_index.cmp(&right.call_index)) + }); + Ok(build_cache_report( + CacheReportInput { + label: query.label.clone(), + since: query.since, + until: query.until, + filters: CacheReportFilters { + provider: query.provider.clone(), + account: query.account.clone(), + session: query.session.clone(), + }, + events: &events, + summaries: &summaries, + calls, + include_details: query.details, + }, + query.timeline.then_some(timeline_zone), + )) + } + + /// Records what an HTTP receiver's preflight said about `cache_health`. + pub fn record_sync_target_cache_health_support( + &self, + sink: &str, + target: &str, + accepted: bool, + ) -> Result<()> { + self.set_metadata_value( + &format!("{SYNC_CACHE_HEALTH_METADATA_PREFIX}{sink}:{target}"), + if accepted { "1" } else { "0" }, + ) + } + + /// Whether summaries for this target keep `cache_health`. An HTTP receiver + /// keeps it only after its preflight advertised support: one that would + /// discard the object must not be sent it, or the summary would look + /// synced with the diagnostics silently lost. Local sinks write what they + /// are given. + pub fn sync_target_accepts_cache_health(&self, sink: &str, target: &str) -> Result { + if sink != "http" { + return Ok(true); + } + Ok(self + .metadata_value(&format!( + "{SYNC_CACHE_HEALTH_METADATA_PREFIX}{sink}:{target}" + ))? + .as_deref() + == Some("1")) + } + + /// Diagnostics for one daily bucket, or `None` for providers the detector + /// does not cover. + pub(crate) fn bucket_cache_health( + &self, + key: &SyncRollupBucketKey, + events: &[UsageEvent], + memo: &mut CacheSessionMemo, + ) -> Result> { + if !provider_has_cache_diagnostics(&key.provider) { + return Ok(None); + } + let mut health = CacheHealthV1::default(); + for event in events { + health.add_event_input(event); + let session = self.analyzed_session(&event.session.session_id, memo)?; + for call in session.get(&event.event_id.0).into_iter().flatten() { + health.add_call(call); + } + } + Ok(Some(health)) + } + + /// Buckets holding calls that may follow a changed event in its stream. + /// + /// A call's verdict depends on its predecessor, so a late arrival, a + /// correction, or a deletion can change the bucket of the next call, which + /// may be a later day. Every event of the session at or after the earliest + /// change is included, found as a range of the session index; refreshing + /// an unchanged bucket writes nothing. + pub(crate) fn cache_successor_buckets( + &self, + floors: &BTreeMap>, + ) -> Result> { + let sql = format!( + "SELECT {CACHE_BUCKET_COLUMNS_SQL} FROM usage_events + WHERE {EVENT_SESSION_ID_SQL} = ?1 AND started_at >= ?2" + ); + let mut statement = self.conn.prepare(&sql)?; + let mut buckets = BTreeSet::new(); + for (session_id, floor) in floors { + let rows = + statement.query_map(params![session_id, floor.to_rfc3339()], cache_bucket_key)?; + for key in rows { + buckets.extend(key?.filter(|key| provider_has_cache_diagnostics(&key.provider))); + } + } + Ok(buckets) + } + + /// Rebuilds the daily summaries this report changes, once per detector + /// version: those of the covered providers, so history whose transcripts + /// are gone gains diagnostics too, and any other provider's that hold cache + /// writes, whose hit ratio now counts them. Summaries whose content is + /// unchanged are left alone, so nothing is queued for sync unless its + /// payload actually changed. + pub(crate) fn ensure_cache_health_rollups(&self) -> Result<()> { + let version = CACHE_HEALTH_VERSION.to_string(); + if self + .metadata_value(CACHE_HEALTH_ROLLUPS_METADATA_KEY)? + .as_deref() + == Some(version.as_str()) + { + return Ok(()); + } + let keys = self.cache_report_bucket_keys()?; + self.with_immediate_transaction(|| { + self.refresh_sync_rollups_for_keys(&keys)?; + self.set_metadata_value(CACHE_HEALTH_ROLLUPS_METADATA_KEY, &version) + }) + } + + fn cache_report_bucket_keys(&self) -> Result> { + let placeholders = sqlite_in_clause_placeholders(CACHE_DIAGNOSTIC_PROVIDERS.len()); + let sql = format!( + "SELECT {CACHE_BUCKET_COLUMNS_SQL} FROM usage_events + WHERE provider IN ({placeholders}) + OR (json_valid(payload) + AND json_extract(payload, '$.usage.cache_creation_tokens') > 0)" + ); + let mut statement = self.conn.prepare(&sql)?; + let rows = statement.query_map( + rusqlite::params_from_iter(CACHE_DIAGNOSTIC_PROVIDERS), + cache_bucket_key, + )?; + let mut keys = BTreeSet::new(); + for key in rows { + keys.extend(key?); + } + Ok(keys) + } + + fn analyzed_session<'memo>( + &self, + session_id: &str, + memo: &'memo mut CacheSessionMemo, + ) -> Result<&'memo AnalyzedSession> { + if !memo.sessions.contains_key(session_id) { + let events = self.cache_session_events(session_id)?; + let mut calls_by_event = AnalyzedSession::new(); + for call in analyze_and_price(&events) { + calls_by_event + .entry(call.event_id.0.clone()) + .or_default() + .push(call); + } + if memo.order.len() >= CACHE_SESSION_MEMO_CAPACITY { + if let Some(evicted) = memo.order.pop_front() { + memo.sessions.remove(&evicted); + } + } + memo.order.push_back(session_id.to_string()); + memo.sessions.insert(session_id.to_string(), calls_by_event); + } + Ok(&memo.sessions[session_id]) + } + + /// Every event of a session, in time order. + fn cache_session_events(&self, session_id: &str) -> Result> { + let sql = format!( + "SELECT payload FROM usage_events WHERE {EVENT_SESSION_ID_SQL} = ?1 + ORDER BY started_at, event_id" + ); + let mut statement = self.conn.prepare(&sql)?; + let rows = statement.query_map(params![session_id], |row| row.get::<_, String>(0))?; + let mut events = Vec::new(); + for row in rows { + events.push(serde_json::from_str(&row?)?); + } + Ok(events) + } + + /// Every event of a session, each marked with whether it starts inside the + /// report's period. The bounds compare as `events_in_period` compares them. + fn cache_session_events_in_period( + &self, + session_id: &str, + query: &CacheReportQuery, + ) -> Result> { + let sql = format!( + "SELECT payload, (?2 IS NULL OR started_at >= ?2) AND started_at <= ?3 + FROM usage_events WHERE {EVENT_SESSION_ID_SQL} = ?1 + ORDER BY started_at, event_id" + ); + let mut statement = self.conn.prepare(&sql)?; + let rows = statement.query_map( + params![ + session_id, + query.since.map(|since| since.to_rfc3339()), + query.until.to_rfc3339() + ], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, bool>(1)?)), + )?; + let mut events = Vec::new(); + for row in rows { + let (payload, in_period) = row?; + events.push((serde_json::from_str(&payload)?, in_period)); + } + Ok(events) + } + + /// Sessions of the covered providers with an event in the report's period + /// that its provider and session filters allow. + fn cache_report_sessions(&self, query: &CacheReportQuery) -> Result> { + let providers = CACHE_DIAGNOSTIC_PROVIDERS + .iter() + .filter(|provider| { + query + .provider + .as_deref() + .is_none_or(|filter| filter == **provider) + }) + .map(|provider| (*provider).to_string()) + .collect::>(); + if providers.is_empty() { + return Ok(Vec::new()); + } + let mut bounds = vec![query.until.to_rfc3339()]; + let since_sql = match query.since { + Some(since) => { + bounds.push(since.to_rfc3339()); + "AND started_at >= ?" + } + None => "", + }; + let sql = format!( + "SELECT DISTINCT {EVENT_SESSION_ID_SQL} FROM usage_events + WHERE started_at <= ? {since_sql} AND provider IN ({})", + sqlite_in_clause_placeholders(providers.len()) + ); + let mut statement = self.conn.prepare(&sql)?; + let rows = statement.query_map( + rusqlite::params_from_iter(bounds.iter().chain(&providers)), + |row| row.get::<_, Option>(0), + )?; + let mut sessions = Vec::new(); + for session_id in rows { + let Some(session_id) = session_id? else { + continue; + }; + if query + .session + .as_deref() + .is_none_or(|filter| session_id.starts_with(filter)) + { + sessions.push(session_id); + } + } + Ok(sessions) + } + + /// Events in the report's period from providers without diagnostics. + fn cache_report_other_events(&self, query: &CacheReportQuery) -> Result> { + if query + .provider + .as_deref() + .is_some_and(provider_has_cache_diagnostics) + { + return Ok(Vec::new()); + } + let mut bounds = vec![query.until.to_rfc3339()]; + let since_sql = match query.since { + Some(since) => { + bounds.push(since.to_rfc3339()); + "AND started_at >= ?" + } + None => "", + }; + let sql = format!( + "SELECT payload FROM usage_events + WHERE started_at <= ? {since_sql} AND provider NOT IN ({})", + sqlite_in_clause_placeholders(CACHE_DIAGNOSTIC_PROVIDERS.len()) + ); + let mut statement = self.conn.prepare(&sql)?; + let rows = statement.query_map( + rusqlite::params_from_iter( + bounds + .iter() + .map(String::as_str) + .chain(CACHE_DIAGNOSTIC_PROVIDERS.iter().copied()), + ), + |row| row.get::<_, String>(0), + )?; + let mut events = Vec::new(); + for row in rows { + events.push(serde_json::from_str(&row?)?); + } + Ok(events) + } +} + +/// A summary in the form a target stores it. Selection and the recorded +/// payload hash both see this form, so a receiver without `cache_health` +/// support is not re-sent summaries that differ only by it. +pub fn summary_for_sync_target(summary: UsageSummary, accepts_cache_health: bool) -> UsageSummary { + if accepts_cache_health { + summary + } else { + summary.without_cache_health() + } +} + +/// A summary that starts within the report's range. The dashboard's period +/// views count a summary the same way, including one that spans several days. +fn cache_report_includes_summary(query: &CacheReportQuery, summary: &UsageSummary) -> bool { + let (start, _) = summary_period_bounds(summary); + let in_range = query.since.is_none_or(|since| start >= since) && start <= query.until; + let provider_matches = query + .provider + .as_deref() + .is_none_or(|provider| summary.provider == provider); + let account_matches = + query + .account + .as_deref() + .is_none_or(|account| match summary.provider_account_id.as_ref() { + Some(id) => id.0 == account, + None => account == CACHE_REPORT_UNASSIGNED_ACCOUNT, + }); + in_range && provider_matches && account_matches +} + +fn cache_report_includes(query: &CacheReportQuery, event: &UsageEvent) -> bool { + let provider_matches = query + .provider + .as_deref() + .is_none_or(|provider| event.provider == provider); + let account_matches = + query + .account + .as_deref() + .is_none_or(|account| match event.provider_account_id.as_ref() { + Some(id) => id.0 == account, + None => account == CACHE_REPORT_UNASSIGNED_ACCOUNT, + }); + let session_matches = query + .session + .as_deref() + .is_none_or(|session| event.session.session_id.starts_with(session)); + provider_matches && account_matches && session_matches +} + +/// Classifies a session's calls and prices the reuse each suspected loss +/// missed. +fn analyze_and_price(events: &[UsageEvent]) -> Vec { + let by_id = events + .iter() + .map(|event| (event.event_id.0.as_str(), event)) + .collect::>(); + let mut calls = analyze_cache_calls(events); + for call in &mut calls { + let (Some(missed), Some(event)) = ( + call.verdict.missed_tokens, + by_id.get(call.event_id.0.as_str()), + ) else { + continue; + }; + call.verdict.missed_cost_micro_usd = missed_reuse_cost(event, call, missed); + } + calls +} + +/// The call's cost at API prices minus its cost had `missed` tokens been +/// cache reads. A lost cache is written again, so they come out of the call's +/// cache writes first, then out of its ordinary input. `None` when the model +/// has no known pricing. +fn missed_reuse_cost(event: &UsageEvent, call: &AnalyzedCacheCall, missed: u64) -> Option { + // A call inside an aggregated event has no split of its writes by cache + // lifetime, and is priced by its own model alone: the event's model + // identity, such as its provider model id, belongs to the turn's last call. + let single_call = event + .context + .as_ref() + .is_none_or(|context| context.calls.is_empty()); + let model = match call.model.as_deref() { + Some(name) if !single_call => Some(statsai_core::ModelInfo { + name: Some(name.to_string()), + normalized_name: Some(name.to_string()), + ..statsai_core::ModelInfo::default() + }), + _ => event.model.clone(), + }; + // One request, so per-request pricing such as long-context rates applies. + let actual = UsageCounts { + input_tokens: Some(call.input_tokens), + cache_read_tokens: Some(call.cache_read_tokens), + cache_creation_tokens: call.cache_creation_tokens, + cache_creation_5m_tokens: event.usage.cache_creation_5m_tokens.filter(|_| single_call), + cache_creation_1h_tokens: event.usage.cache_creation_1h_tokens.filter(|_| single_call), + requests: Some(1), + ..UsageCounts::default() + }; + let reused = usage_with_reads(&actual, missed); + let cost = |usage: &UsageCounts| { + statsai_pricing::estimate_cost_at(&call.provider, model.as_ref(), usage, &call.completed_at) + .estimated_api_equivalent_micro_usd + }; + let difference = cost(&actual)?.saturating_sub(cost(&reused)?); + Some(u64::try_from(difference).unwrap_or(0)) +} + +/// `usage` with `tokens` moved into cache reads from cache writes, keeping +/// the split by cache lifetime in proportion, and then from ordinary input. +fn usage_with_reads(usage: &UsageCounts, tokens: u64) -> UsageCounts { + let writes = usage.cache_creation_tokens.unwrap_or(0); + let from_writes = tokens.min(writes); + let from_input = (tokens - from_writes).min(usage.input_tokens.unwrap_or(0)); + let shrink = |part: Option| { + part.map(|part| { + let removed = u128::from(part) * u128::from(from_writes) / u128::from(writes.max(1)); + part.saturating_sub(u64::try_from(removed).unwrap_or(part)) + }) + }; + UsageCounts { + input_tokens: usage.input_tokens.map(|input| input - from_input), + cache_read_tokens: Some( + usage + .cache_read_tokens + .unwrap_or(0) + .saturating_add(from_writes + from_input), + ), + cache_creation_tokens: usage.cache_creation_tokens.map(|_| writes - from_writes), + cache_creation_5m_tokens: shrink(usage.cache_creation_5m_tokens), + cache_creation_1h_tokens: shrink(usage.cache_creation_1h_tokens), + ..usage.clone() + } +} + +/// The daily bucket of a row of [`CACHE_BUCKET_COLUMNS_SQL`], or `None` when +/// its start time does not parse. +fn cache_bucket_key(row: &rusqlite::Row<'_>) -> rusqlite::Result> { + let started_at = row.get::<_, String>(3)?; + let Ok(started_at) = DateTime::parse_from_rfc3339(&started_at) else { + return Ok(None); + }; + let project = row.get::<_, Option>(4)?; + let project = project + .as_deref() + .and_then(|project| serde_json::from_str::(project).ok()); + Ok(Some(SyncRollupBucketKey { + provider: row.get(0)?, + source_id: row.get(1)?, + provider_account_id: row.get(2)?, + day_key: started_at.with_timezone(&Utc).date_naive().to_string(), + project_key: sync_rollup_project_key(project.as_ref()), + })) +} diff --git a/crates/statsai-store/src/events.rs b/crates/statsai-store/src/events.rs index 92483f8..f15dc7e 100644 --- a/crates/statsai-store/src/events.rs +++ b/crates/statsai-store/src/events.rs @@ -5,6 +5,9 @@ use crate::dedupe::*; pub(crate) struct EventWriteDirty { pub(crate) buckets: BTreeSet, pub(crate) sessions: BTreeSet, + /// Earliest changed event per session, for providers with prompt-cache + /// diagnostics. Later calls in the session may need their verdicts redone. + pub(crate) cache_floors: BTreeMap>, } impl EventWriteDirty { @@ -17,11 +20,24 @@ impl EventWriteDirty { fn observe(&mut self, event: &UsageEvent) { self.buckets.insert(sync_rollup_bucket_key(event)); self.sessions.insert(event.session.session_id.clone()); + if statsai_core::provider_has_cache_diagnostics(&event.provider) { + self.observe_cache_floor(&event.session.session_id, event.session.started_at); + } + } + + fn observe_cache_floor(&mut self, session_id: &str, started_at: DateTime) { + self.cache_floors + .entry(session_id.to_string()) + .and_modify(|floor| *floor = (*floor).min(started_at)) + .or_insert(started_at); } pub(crate) fn extend(&mut self, other: Self) { self.buckets.extend(other.buckets); self.sessions.extend(other.sessions); + for (session_id, started_at) in other.cache_floors { + self.observe_cache_floor(&session_id, started_at); + } } } @@ -164,8 +180,10 @@ impl Store { }) } - fn refresh_event_rollups(&self, dirty: &EventWriteDirty) -> Result<()> { - self.refresh_sync_rollups_for_keys(&dirty.buckets)?; + pub(crate) fn refresh_event_rollups(&self, dirty: &EventWriteDirty) -> Result<()> { + let mut buckets = self.cache_successor_buckets(&dirty.cache_floors)?; + buckets.extend(dirty.buckets.iter().cloned()); + self.refresh_sync_rollups_for_keys(&buckets)?; self.refresh_session_rollups_for_keys(&dirty.sessions)?; Ok(()) } diff --git a/crates/statsai-store/src/lib.rs b/crates/statsai-store/src/lib.rs index 1ca46a8..fdb4aac 100644 --- a/crates/statsai-store/src/lib.rs +++ b/crates/statsai-store/src/lib.rs @@ -4,6 +4,7 @@ mod account_plan; mod accounts; mod activity; mod archive; +mod cache; mod code_changes; mod dedupe; mod events; @@ -49,18 +50,18 @@ use chrono::{DateTime, Datelike, NaiveDate, Utc}; use rusqlite::{params, Connection, OptionalExtension}; use serde::{Deserialize, Serialize}; use statsai_core::{ - daily_rollup_project_key, hash_text, micro_usd_to_cents_rounded, normalize_email, - normalize_provider_user_id, periods_overlap, project_contains_file_paths, + cache_hit_ratio, daily_rollup_project_key, hash_text, micro_usd_to_cents_rounded, + normalize_email, normalize_provider_user_id, periods_overlap, project_contains_file_paths, project_has_stable_identity, provider_account_id, provider_account_id_from_identity, sanitize_code_change_metric_for_sync, sanitize_summary_for_sync, semantic_event_fingerprint, source_account_assignment_id, subscription_id, summary_id, timestamp_in_period, AccountEvidenceSummaryV1, AccountPlanProjectionV1, ActivityCoverageV1, ActivityInvocationV1, - BillingPeriod, CodeChangeMetric, Confidence, CostAccumulator, CostInfo, DailyRollup, EventId, - EventSource, IdentitySource, LatencySource, MetricStats, ModelInfo, PrivacyInfo, PrivacyMode, - ProviderAccount, ProviderAccountId, SemanticFingerprintInput, SourceAccountAssignment, - SourceAccountAssignmentId, SourceId, SourceKind, SourceLocation, SourceVerificationMode, - Subscription, SubscriptionId, SubscriptionStatus, SummaryId, SummaryMetadata, - SummaryMetricTotals, SummaryMetrics, SummaryModelMetrics, SummaryModelUsage, + BillingPeriod, CacheHealthV1, CodeChangeMetric, Confidence, CostAccumulator, CostInfo, + DailyRollup, EventId, EventSource, IdentitySource, LatencySource, MetricStats, ModelInfo, + PrivacyInfo, PrivacyMode, ProviderAccount, ProviderAccountId, SemanticFingerprintInput, + SourceAccountAssignment, SourceAccountAssignmentId, SourceId, SourceKind, SourceLocation, + SourceVerificationMode, Subscription, SubscriptionId, SubscriptionStatus, SummaryId, + SummaryMetadata, SummaryMetricTotals, SummaryMetrics, SummaryModelMetrics, SummaryModelUsage, SyncAuthoritativeSnapshot, SyncBatch, TaskVerificationCursor, TaskVerificationId, UsageCounts, UsageEvent, UsageSummary, VerifiedSourceObservation, VerifiedSourceState, VerifiedSubscriptionState, PROVIDER_ACCOUNT_SCHEMA_VERSION, @@ -96,6 +97,8 @@ const INFERRED_SOURCE_OBSERVATION_HASH_PREFIX: &str = "inferred_source.v1:"; const VERIFIED_SOURCE_OBSERVATION_HASH_PREFIX: &str = "verified_source.v2:"; pub use archive::{ArchiveConversationSummary, ArchiveSearchHit, ArchiveStats, ArchiveWriteResult}; +pub(crate) use cache::CacheSessionMemo; +pub use cache::{summary_for_sync_target, CacheReportQuery, CACHE_REPORT_UNASSIGNED_ACCOUNT}; pub use code_changes::CodeChangeRefreshReport; pub use privacy::{ FilteredConversationMetadata, FilteredConversationRecord, PrivacyDatasetStatus, diff --git a/crates/statsai-store/src/migrations/mod.rs b/crates/statsai-store/src/migrations/mod.rs index e944d3c..a769d63 100644 --- a/crates/statsai-store/src/migrations/mod.rs +++ b/crates/statsai-store/src/migrations/mod.rs @@ -8,7 +8,7 @@ mod v2; pub(crate) use v1::*; pub(crate) use v2::*; -pub const CURRENT_SCHEMA_VERSION: i64 = 29; +pub const CURRENT_SCHEMA_VERSION: i64 = 30; pub fn migrate(conn: &Connection) -> Result<()> { if let Some(current) = existing_schema_version(conn)? { @@ -154,6 +154,7 @@ fn apply_migration(conn: &Connection, version: i64) -> Result<()> { 27 => apply_migration_027(conn), 28 => apply_migration_028(conn), 29 => apply_migration_029(conn), + 30 => apply_migration_030(conn), _ => bail!("unsupported schema migration version {version}"), } } @@ -291,7 +292,10 @@ pub fn schema_version(conn: &Connection) -> Result { #[cfg(test)] mod tests { use super::*; - use crate::{EVENT_CONVERSATION_HASH_SQL, EVENT_SOURCE_FILE_HASH_SQL, QUOTA_PLAN_TYPE_SQL}; + use crate::{ + EVENT_CONVERSATION_HASH_SQL, EVENT_SESSION_ID_SQL, EVENT_SOURCE_FILE_HASH_SQL, + QUOTA_PLAN_TYPE_SQL, + }; use rusqlite::Connection; #[test] @@ -356,6 +360,40 @@ mod tests { &conn, "usage_events_provider_account_started_idx" )); + assert!(index_exists(&conn, "usage_events_session_started_idx")); + assert!(!index_exists(&conn, "usage_events_session_idx")); + } + + /// Session reads come back in time order straight from the index, and the + /// calls after a changed event are a range of it, so neither sorts nor + /// scans the session. + #[test] + fn session_reads_are_ordered_ranges_of_the_session_index() { + let conn = Connection::open_in_memory().expect("open in-memory database"); + migrate(&conn).expect("migrate fresh database"); + + for (label, sql) in [ + ( + "read a session in order", + format!( + "SELECT payload FROM usage_events WHERE {EVENT_SESSION_ID_SQL} = ?1 + ORDER BY started_at, event_id" + ), + ), + ( + "find the events after a changed one", + format!( + "SELECT provider FROM usage_events + WHERE {EVENT_SESSION_ID_SQL} = ?1 AND started_at >= ?2" + ), + ), + ] { + let plan = query_plan(&conn, &sql); + assert!( + plan.contains("usage_events_session_started_idx") && !plan.contains("TEMP B-TREE"), + "{label} does not read the session index in order: {plan}" + ); + } } /// The scan filters are only fast while SQLite can match them to the indexes diff --git a/crates/statsai-store/src/migrations/v2.rs b/crates/statsai-store/src/migrations/v2.rs index 021e46f..67c475c 100644 --- a/crates/statsai-store/src/migrations/v2.rs +++ b/crates/statsai-store/src/migrations/v2.rs @@ -684,3 +684,22 @@ pub(crate) fn apply_migration_029(conn: &Connection) -> Result<()> { )?; Ok(()) } + +/// Orders each session's events by time in the session index. +/// +/// The prompt-cache detector judges each call against the one before it in +/// its session, so it reads sessions whole, and after a late arrival or a +/// correction it looks up the session's events at or after the changed one. +/// Leading with the session and then `started_at` serves both as index range +/// scans, and gives session reads their order without a sort. It replaces +/// `usage_events_session_idx`, whose session lookups it also covers. +pub(crate) fn apply_migration_030(conn: &Connection) -> Result<()> { + conn.execute_batch(&format!( + r#" + CREATE INDEX IF NOT EXISTS usage_events_session_started_idx + ON usage_events ({EVENT_SESSION_ID_SQL}, started_at, event_id); + DROP INDEX IF EXISTS usage_events_session_idx; + "#, + ))?; + Ok(()) +} diff --git a/crates/statsai-store/src/pricing/tests/repricing.rs b/crates/statsai-store/src/pricing/tests/repricing.rs index 20b9f8b..1f5ca3b 100644 --- a/crates/statsai-store/src/pricing/tests/repricing.rs +++ b/crates/statsai-store/src/pricing/tests/repricing.rs @@ -384,7 +384,7 @@ fn daily_rollups_are_unused_by_report_and_sync_paths() { assert_eq!(events[0].cost, expected_review_cost(started_at)); // Pin: bump this when CURRENT_SCHEMA_VERSION changes, after confirming // the daily_rollups table is still unused by report/sync/snapshot. - assert_eq!(CURRENT_SCHEMA_VERSION, 29); + assert_eq!(CURRENT_SCHEMA_VERSION, 30); } #[test] diff --git a/crates/statsai-store/src/pricing/tests/support.rs b/crates/statsai-store/src/pricing/tests/support.rs index 828b453..02abc63 100644 --- a/crates/statsai-store/src/pricing/tests/support.rs +++ b/crates/statsai-store/src/pricing/tests/support.rs @@ -87,6 +87,7 @@ pub(super) fn test_event( }, created_at: started_at, imported_at: started_at, + context: None, } } diff --git a/crates/statsai-store/src/quota/tests/support.rs b/crates/statsai-store/src/quota/tests/support.rs index c4e634d..c681f3f 100644 --- a/crates/statsai-store/src/quota/tests/support.rs +++ b/crates/statsai-store/src/quota/tests/support.rs @@ -172,5 +172,6 @@ pub(super) fn sample_usage_event( }, created_at: started_at, imported_at: started_at, + context: None, } } diff --git a/crates/statsai-store/src/rollups/build.rs b/crates/statsai-store/src/rollups/build.rs index 30bb3a5..0908e36 100644 --- a/crates/statsai-store/src/rollups/build.rs +++ b/crates/statsai-store/src/rollups/build.rs @@ -48,7 +48,10 @@ pub(crate) fn event_with_valid_project(event: &UsageEvent) -> UsageEvent { event } -pub(crate) fn build_sync_rollup_summary(events: &[UsageEvent]) -> UsageSummary { +pub(crate) fn build_sync_rollup_summary( + events: &[UsageEvent], + cache_health: Option, +) -> UsageSummary { let first = events.first().expect("rollup bucket must contain events"); // Events arrive oldest first. A bucket can now span a repository rename, so // project metadata comes from the newest event: it names the remote the @@ -177,14 +180,12 @@ pub(crate) fn build_sync_rollup_summary(events: &[UsageEvent]) -> UsageSummary { tracked_reasoning_tokens.saturating_add(event.usage.reasoning_tokens.unwrap_or(0)); } - let prompt_tokens = event - .usage - .input_tokens - .unwrap_or(0) - .saturating_add(event.usage.cache_read_tokens.unwrap_or(0)); - if prompt_tokens > 0 { - cache_hit_ratio_values - .push(event.usage.cache_read_tokens.unwrap_or(0) as f64 / prompt_tokens as f64); + if let Some(ratio) = cache_hit_ratio( + event.usage.input_tokens.unwrap_or(0), + event.usage.cache_creation_tokens.unwrap_or(0), + event.usage.cache_read_tokens.unwrap_or(0), + ) { + cache_hit_ratio_values.push(ratio); } let generated_tokens = event .usage @@ -318,6 +319,7 @@ pub(crate) fn build_sync_rollup_summary(events: &[UsageEvent]) -> UsageSummary { user_messages: (user_messages > 0).then_some(user_messages), assistant_messages: (assistant_messages > 0).then_some(assistant_messages), developer_messages: (developer_messages > 0).then_some(developer_messages), + cache_health, }); let total_sessions = (!session_ids.is_empty()).then_some(session_ids.len() as u64); let total_messages_metadata = summary_metrics @@ -424,23 +426,7 @@ fn percentile_nearest_rank(values: &[f64], percentile: f64) -> Option { } fn summary_metrics_or_none(metrics: SummaryMetrics) -> Option { - let has_metrics = metrics.active_seconds.is_some() - || metrics.tracked_requests.is_some() - || metrics.tracked_output_tokens.is_some() - || metrics.tracked_reasoning_tokens.is_some() - || metrics.latency_ms.is_some() - || metrics.time_to_first_token_ms.is_some() - || metrics.generated_tps.is_some() - || metrics.visible_tps.is_some() - || metrics.overall_generated_tps.is_some() - || metrics.overall_visible_tps.is_some() - || metrics.cache_hit_ratio.is_some() - || metrics.reasoning_share.is_some() - || metrics.total_messages.is_some() - || metrics.user_messages.is_some() - || metrics.assistant_messages.is_some() - || metrics.developer_messages.is_some(); - has_metrics.then_some(metrics) + (!metrics.is_empty()).then_some(metrics) } #[derive(Debug, Default)] diff --git a/crates/statsai-store/src/rollups/mod.rs b/crates/statsai-store/src/rollups/mod.rs index 38807a1..339934f 100644 --- a/crates/statsai-store/src/rollups/mod.rs +++ b/crates/statsai-store/src/rollups/mod.rs @@ -84,7 +84,7 @@ impl Store { if stale_count > 0 { self.rebuild_sync_rollups()?; } - Ok(()) + self.ensure_cache_health_rollups() } pub fn sync_rollup_period_stats(&self, cutoff_day: NaiveDate) -> Result { @@ -257,10 +257,16 @@ impl Store { } fn pending_sync_rollup_counts(&self, sink: &str, target: &str) -> Result<(u64, u64)> { + let accepts_cache_health = self.sync_target_accepts_cache_health(sink, target)?; let rollups = self .all_sync_rollup_summaries()? .into_iter() - .map(sanitize_summary_for_default_http_sync) + .map(|summary| { + summary_for_sync_target( + sanitize_summary_for_default_http_sync(summary), + accepts_cache_health, + ) + }) .collect::>(); let pending = self.pending_summaries_for_sync(sink, target, &rollups)?; let days = collect_pending_summary_days(pending.iter()); @@ -269,7 +275,12 @@ impl Store { pub fn reconcile_sync_rollup_dirty_flags(&self, sink: &str, target: &str) -> Result { self.ensure_current_sync_rollup_versions()?; - let summaries = self.all_sync_rollup_summaries()?; + let accepts_cache_health = self.sync_target_accepts_cache_health(sink, target)?; + let summaries = self + .all_sync_rollup_summaries()? + .into_iter() + .map(|summary| summary_for_sync_target(summary, accepts_cache_health)) + .collect::>(); self.with_immediate_transaction(|| { self.reconcile_sync_rollup_dirty_flags_in_transaction(sink, target, &summaries) }) @@ -470,16 +481,21 @@ impl Store { &self, keys: &BTreeSet, ) -> Result { + let mut memo = CacheSessionMemo::default(); let mut refreshed = 0u64; for key in keys { - if self.refresh_sync_rollup_for_key(key)? { + if self.refresh_sync_rollup_for_key(key, &mut memo)? { refreshed += 1; } } Ok(refreshed) } - fn refresh_sync_rollup_for_key(&self, key: &SyncRollupBucketKey) -> Result { + fn refresh_sync_rollup_for_key( + &self, + key: &SyncRollupBucketKey, + memo: &mut CacheSessionMemo, + ) -> Result { let events = self.sync_rollup_events(key)?; if events.is_empty() { let deleted = self.conn.execute( @@ -489,7 +505,8 @@ impl Store { return Ok(deleted > 0); } - let summary = build_sync_rollup_summary(&events); + let cache_health = self.bucket_cache_health(key, &events, memo)?; + let summary = build_sync_rollup_summary(&events, cache_health); let payload = serde_json::to_string(&summary)?; let payload_hash = summary_sync_payload_hash(&summary)?; let existing: Option<(String, i64)> = self diff --git a/crates/statsai-store/src/sync_state/mod.rs b/crates/statsai-store/src/sync_state/mod.rs index a19be32..28018da 100644 --- a/crates/statsai-store/src/sync_state/mod.rs +++ b/crates/statsai-store/src/sync_state/mod.rs @@ -563,7 +563,12 @@ impl Store { .filter(|summary| is_daily_rollup_summary(summary)) .map(|summary| summary.summary_id.clone()) .collect(); - let rollup_summaries = self.all_sync_rollup_summaries()?; + let accepts_cache_health = self.sync_target_accepts_cache_health(sink, target)?; + let rollup_summaries = self + .all_sync_rollup_summaries()? + .into_iter() + .map(|summary| summary_for_sync_target(summary, accepts_cache_health)) + .collect::>(); let task_verification_cursor = Self::sync_batch_task_verification_cursor(batch); self.with_immediate_transaction(|| { diff --git a/crates/statsai-store/src/sync_state/pending.rs b/crates/statsai-store/src/sync_state/pending.rs index 8c38c2b..d0bcc85 100644 --- a/crates/statsai-store/src/sync_state/pending.rs +++ b/crates/statsai-store/src/sync_state/pending.rs @@ -421,10 +421,16 @@ impl Store { target: &str, include_projects: bool, ) -> Result> { + let accepts_cache_health = self.sync_target_accepts_cache_health("http", target)?; let rollups = self .all_sync_rollup_summaries()? .into_iter() - .map(|summary| sanitize_summary_for_http_sync(summary, include_projects)) + .map(|summary| { + summary_for_sync_target( + sanitize_summary_for_http_sync(summary, include_projects), + accepts_cache_health, + ) + }) .collect::>(); self.pending_summaries_for_sync("http", target, &rollups) } @@ -443,17 +449,28 @@ impl Store { device_id: &str, include_projects: bool, ) -> Result { + let accepts_cache_health = self.sync_target_accepts_cache_health("http", target)?; let current_rollups = self .all_sync_rollup_summaries()? .into_iter() - .map(|summary| sanitize_summary_for_http_sync(summary, include_projects)) + .map(|summary| { + summary_for_sync_target( + sanitize_summary_for_http_sync(summary, include_projects), + accepts_cache_health, + ) + }) .collect::>(); let rollups = self.pending_summaries_for_sync("http", target, ¤t_rollups)?; let current_passthrough_summaries = self .summaries()? .into_iter() .filter(is_http_rollup_passthrough_summary) - .map(|summary| sanitize_summary_for_http_sync(summary, include_projects)) + .map(|summary| { + summary_for_sync_target( + sanitize_summary_for_http_sync(summary, include_projects), + accepts_cache_health, + ) + }) .collect::>(); let passthrough_summaries = self.pending_summaries_for_sync("http", target, ¤t_passthrough_summaries)?; diff --git a/crates/statsai-store/src/tests/cache.rs b/crates/statsai-store/src/tests/cache.rs new file mode 100644 index 0000000..4ca1c2f --- /dev/null +++ b/crates/statsai-store/src/tests/cache.rs @@ -0,0 +1,636 @@ +use super::support::*; +use super::*; +use statsai_core::{ + CacheHealthV1, CallContext, EventId, IdentitySource, ModelCall, CACHE_HEALTH_VERSION, +}; + +fn claude_source() -> SourceLocation { + SourceLocation::local_adapter( + "claude_code", + "test", + "0", + Path::new("/fixture/claude"), + LocationOrigin::Configured, + ) +} + +fn at(day: u32, hour: u32, minute: u32) -> chrono::DateTime { + Utc.with_ymd_and_hms(2026, 7, day, hour, minute, 0) + .single() + .expect("time") +} + +/// One Claude Code call in session `session_cache`, transcript `main`. +fn call( + source: &SourceLocation, + index: u64, + time: chrono::DateTime, + input: u64, + read: u64, + write: u64, +) -> UsageEvent { + let mut event = test_store_event(source, time, &format!("call-{index}")); + event.provider = "claude_code".to_string(); + event.event_id = EventId(format!("event_cache_{index:03}")); + event.session.session_id = "session_cache".to_string(); + event.usage = UsageCounts { + input_tokens: Some(input), + cache_read_tokens: Some(read), + cache_creation_tokens: Some(write), + output_tokens: Some(10), + total_tokens: Some(input + read + write + 10), + requests: Some(1), + ..UsageCounts::default() + }; + event.parse_evidence = Some(ParseEvidence { + event_key_version: "test".to_string(), + source_file_path_hash: Some("main".to_string()), + source_line_number: Some(index), + source_record_id: None, + model_inferred: false, + timestamp_inferred: false, + account_identity_source: IdentitySource::Unresolved, + }); + event +} + +fn store_with_source() -> (Store, SourceLocation) { + let store = Store::in_memory().expect("store"); + let source = claude_source(); + store.upsert_source(&source).expect("source"); + (store, source) +} + +fn health_by_day(store: &Store) -> BTreeMap { + store + .all_sync_rollup_summaries() + .expect("summaries") + .into_iter() + .filter_map(|summary| { + let day = summary.period_start?.date_naive().to_string(); + Some((day, summary.metrics?.cache_health?)) + }) + .collect() +} + +fn payload_hashes(store: &Store) -> BTreeMap { + let mut statement = store + .conn + .prepare("SELECT summary_id, payload_hash, updated_at FROM sync_rollups") + .expect("statement"); + statement + .query_map([], |row| Ok((row.get(0)?, (row.get(1)?, row.get(2)?)))) + .expect("rows") + .collect::>() + .expect("hashes") +} + +#[test] +fn daily_summaries_carry_cache_health_for_covered_providers_only() { + let (store, source) = store_with_source(); + let mut events = vec![ + call(&source, 1, at(1, 10, 0), 10, 0, 30_000), + call(&source, 2, at(1, 10, 1), 10, 30_000, 500), + call(&source, 3, at(1, 10, 20), 10, 0, 30_600), + ]; + let mut cursor = test_store_event(&source, at(1, 11, 0), "cursor-row"); + cursor.provider = "cursor".to_string(); + events.push(cursor); + store.insert_events(&events).expect("insert"); + + let summaries = store.all_sync_rollup_summaries().expect("summaries"); + let claude = summaries + .iter() + .find(|summary| summary.provider == "claude_code") + .expect("claude summary"); + let health = claude + .metrics + .as_ref() + .and_then(|metrics| metrics.cache_health.as_ref()) + .expect("cache health"); + assert_eq!(health.version, CACHE_HEALTH_VERSION); + assert_eq!( + (health.requests, health.first_observed, health.first_cold), + (3, 1, 1) + ); + assert_eq!( + (health.comparable, health.losses, health.full_losses), + (2, 1, 1) + ); + assert_eq!(health.gap_comparable[1], 1); + assert_eq!(health.gap_losses[4], 1); + // Writes count in the denominator of the per-call ratio. + assert_eq!( + claude + .metrics + .as_ref() + .and_then(|metrics| metrics.cache_hit_ratio.as_ref()) + .and_then(|ratio| ratio.max), + Some(30_000.0 / 30_510.0) + ); + + let cursor = summaries + .iter() + .find(|summary| summary.provider == "cursor") + .expect("cursor summary"); + assert!(cursor + .metrics + .as_ref() + .is_none_or(|metrics| metrics.cache_health.is_none())); + assert!( + serde_json::to_string(claude).expect("json").len() + - serde_json::to_string(&claude.clone().without_cache_health()) + .expect("json") + .len() + < 2_048, + "the synced object stays under 2 KiB" + ); +} + +#[test] +fn a_predecessor_before_midnight_is_compared_with() { + let (store, source) = store_with_source(); + store + .insert_events(&[ + call(&source, 1, at(1, 23, 50), 10, 0, 50_000), + call(&source, 2, at(1, 23, 58), 10, 50_000, 100), + call(&source, 3, at(2, 0, 2), 10, 0, 50_100), + ]) + .expect("insert"); + + let health = health_by_day(&store); + assert_eq!(health["2026-07-01"].first_observed, 1); + let next_day = &health["2026-07-02"]; + assert_eq!( + next_day.first_observed, 0, + "the previous call is yesterday's" + ); + assert_eq!((next_day.comparable, next_day.losses), (1, 1)); + assert_eq!(next_day.gap_losses[1], 1); +} + +#[test] +fn a_late_predecessor_refreshes_the_later_day() { + let (store, source) = store_with_source(); + store + .insert_events(&[call(&source, 3, at(2, 0, 2), 10, 0, 50_100)]) + .expect("later call"); + assert_eq!(health_by_day(&store)["2026-07-02"].first_observed, 1); + let before = payload_hashes(&store); + + store + .insert_events(&[ + call(&source, 1, at(1, 23, 50), 10, 0, 50_000), + call(&source, 2, at(1, 23, 58), 10, 50_000, 100), + ]) + .expect("late arrivals"); + + let health = health_by_day(&store); + assert_eq!(health["2026-07-02"].first_observed, 0); + assert_eq!(health["2026-07-02"].losses, 1); + let after = payload_hashes(&store); + let changed = before + .iter() + .filter(|(id, (hash, _))| after.get(*id).is_some_and(|(next, _)| next != hash)) + .count(); + assert_eq!(changed, 1, "the later day's summary was rebuilt"); +} + +#[test] +fn a_corrected_predecessor_changes_the_verdict_after_it() { + let (store, source) = store_with_source(); + store + .insert_events(&[ + call(&source, 1, at(1, 23, 58), 10, 50_000, 100), + call(&source, 2, at(2, 0, 2), 10, 0, 50_100), + ]) + .expect("insert"); + assert_eq!(health_by_day(&store)["2026-07-02"].losses, 1); + + // The predecessor turns out to have cached almost nothing. + store + .insert_event(&call(&source, 1, at(1, 23, 58), 50_100, 0, 10)) + .expect("correction"); + let next_day = &health_by_day(&store)["2026-07-02"]; + assert_eq!(next_day.losses, 0); + assert_eq!(next_day.below_threshold, 1); +} + +#[test] +fn sessions_split_by_sub_agent_are_compared_within_their_stream() { + let (store, source) = store_with_source(); + let mut agent = call(&source, 2, at(1, 10, 1), 0, 0, 20_000); + agent.context = Some(CallContext { + agent_hash: Some("agent".to_string()), + ..CallContext::default() + }); + store + .insert_events(&[ + call(&source, 1, at(1, 10, 0), 0, 100_000, 1_000), + agent, + call(&source, 3, at(1, 10, 2), 0, 101_000, 1_000), + ]) + .expect("insert"); + let health = &health_by_day(&store)["2026-07-01"]; + assert_eq!((health.first_observed, health.losses), (2, 0)); +} + +#[test] +fn the_one_time_refresh_adds_cache_health_once_and_leaves_other_summaries_alone() { + let (store, source) = store_with_source(); + let mut cursor = test_store_event(&source, at(1, 11, 0), "cursor-row"); + cursor.provider = "cursor".to_string(); + // Another provider's day with cache writes, whose hit ratio now counts them. + let mut opencode = call(&source, 3, at(1, 12, 0), 100, 100, 100); + opencode.provider = "opencode".to_string(); + opencode.event_id = EventId("event_opencode".to_string()); + opencode.session.session_id = "session_opencode".to_string(); + store + .insert_events(&[ + call(&source, 1, at(1, 10, 0), 10, 0, 30_000), + call(&source, 2, at(1, 10, 1), 10, 30_000, 500), + cursor, + opencode, + ]) + .expect("insert"); + store + .all_sync_rollup_summaries() + .expect("marker set on first read"); + let current = payload_hashes(&store); + + // A store written before this version: no cache_health, hit ratios + // without cache writes, and no marker. + for summary in store.all_sync_rollup_summaries().expect("summaries") { + let mut legacy = summary.without_cache_health(); + if legacy.provider == "opencode" { + let metrics = legacy.metrics.as_mut().expect("metrics"); + let ratio = metrics.cache_hit_ratio.as_mut().expect("hit ratio"); + ratio.p50 = Some(0.5); + } + store + .conn + .execute( + "UPDATE sync_rollups SET payload = ?2, payload_hash = ?3, updated_at = 'legacy', dirty = 0 + WHERE summary_id = ?1", + params![ + &legacy.summary_id.0, + serde_json::to_string(&legacy).expect("json"), + summary_sync_payload_hash(&legacy).expect("hash") + ], + ) + .expect("legacy row"); + } + store + .conn + .execute( + "DELETE FROM local_metadata WHERE key = 'sync_rollups.cache_health_version'", + [], + ) + .expect("clear marker"); + + assert_eq!(health_by_day(&store).len(), 1, "the refresh ran"); + let refreshed = payload_hashes(&store); + for (id, (hash, _)) in &refreshed { + assert_eq!(hash, ¤t[id].0, "content matches a fresh build"); + } + let untouched = refreshed + .values() + .filter(|(_, updated_at)| updated_at == "legacy") + .count(); + assert_eq!(untouched, 1, "the cursor summary was not rewritten"); + + // Running again changes nothing. + store.all_sync_rollup_summaries().expect("second read"); + assert_eq!(payload_hashes(&store), refreshed); +} + +#[test] +fn the_report_agrees_with_the_daily_summaries() { + let (store, source) = store_with_source(); + store + .insert_events(&[ + call(&source, 1, at(1, 23, 50), 10, 0, 50_000), + call(&source, 2, at(1, 23, 58), 10, 50_000, 100), + call(&source, 3, at(2, 0, 2), 10, 0, 50_100), + call(&source, 4, at(2, 1, 30), 10, 50_110, 200), + ]) + .expect("insert"); + let mut query = + CacheReportQuery::for_range(Some("2026-07-02"), Some("2026-07-02"), at(3, 0, 0)) + .expect("query"); + query.details = true; + let report = store.cache_report(&query, &Utc).expect("report"); + + let daily = &health_by_day(&store)["2026-07-02"]; + assert_eq!(&report.diagnostics, daily); + assert_eq!(report.totals.requests, 2); + assert_eq!(report.details.as_ref().map(Vec::len), Some(2)); + assert_eq!(report.days.len(), 1); + + let mut filtered = query.clone(); + filtered.account = Some("unassigned".to_string()); + filtered.session = Some("session_ca".to_string()); + assert_eq!( + store + .cache_report(&filtered, &Utc) + .expect("report") + .totals + .requests, + 2 + ); + filtered.provider = Some("codex".to_string()); + assert_eq!( + store + .cache_report(&filtered, &Utc) + .expect("report") + .totals + .requests, + 0 + ); +} + +#[test] +fn the_report_counts_other_providers_in_its_token_totals_only() { + let (store, source) = store_with_source(); + let other_source = SourceLocation::local_adapter( + "opencode", + "test", + "0", + Path::new("/fixture/opencode"), + LocationOrigin::Configured, + ); + store.upsert_source(&other_source).expect("source"); + let mut other = test_store_event(&other_source, at(2, 9, 0), "other"); + other.provider = "opencode".to_string(); + other.usage.requests = Some(1); + store + .insert_events(&[ + call(&source, 1, at(1, 23, 50), 10, 0, 50_000), + call(&source, 2, at(2, 0, 2), 10, 50_000, 100), + other, + ]) + .expect("insert"); + let query = CacheReportQuery::for_range(Some("2026-07-02"), Some("2026-07-02"), at(3, 0, 0)) + .expect("query"); + + let report = store.cache_report(&query, &Utc).expect("report"); + assert_eq!(report.totals.requests, 2); + assert_eq!(report.diagnostics.requests, 1); + assert_eq!(report.diagnostics.comparable, 1); + + let mut other_only = query.clone(); + other_only.provider = Some("opencode".to_string()); + let report = store.cache_report(&other_only, &Utc).expect("report"); + assert_eq!(report.totals.requests, 1); + assert_eq!(report.diagnostics.requests, 0); +} + +#[test] +fn a_receiver_without_cache_health_is_not_offered_summaries_that_differ_only_by_it() { + let (store, source) = store_with_source(); + store + .insert_events(&[ + call(&source, 1, at(1, 10, 0), 10, 0, 30_000), + call(&source, 2, at(1, 10, 1), 10, 30_000, 500), + ]) + .expect("insert"); + let target = "https://receiver.test/api/sync/batches"; + assert!(!store + .sync_target_accepts_cache_health("http", target) + .expect("default")); + assert!(store + .sync_target_accepts_cache_health("file", "out.json") + .expect("local sink")); + + let stripped = store + .all_sync_rollup_summaries() + .expect("summaries") + .into_iter() + .map(|summary| { + summary_for_sync_target(sanitize_summary_for_default_http_sync(summary), false) + }) + .collect::>(); + assert!(stripped.iter().all(|summary| summary + .metrics + .as_ref() + .is_none_or(|metrics| metrics.cache_health.is_none()))); + store + .record_summaries_synced("http", target, &stripped) + .expect("record"); + assert_eq!( + store + .pending_http_sync_summary_counts_with_projects(target, "device", false) + .expect("pending") + .rollups, + 0 + ); + + store + .record_sync_target_cache_health_support("http", target, true) + .expect("support"); + assert_eq!( + store + .pending_http_sync_summary_counts_with_projects(target, "device", false) + .expect("pending") + .rollups, + 1, + "support turning on queues the diagnostics once" + ); +} + +#[test] +fn the_report_counts_a_multi_day_summary_by_its_start_day() { + let (store, source) = store_with_source(); + let backfill = |key: &str, start: chrono::DateTime, reads: u64| { + let mut summary = test_store_summary(&source, start, reads); + summary.summary_id = summary_id(&source.provider, &source.source_id, key); + summary.source.source_kind = SourceKind::Manual; + summary.metadata.summary_format = "manual_period_summary".to_string(); + summary.period_start = Some(start); + summary.period_end = Some(start + chrono::Duration::days(5)); + summary.usage = UsageCounts { + cache_read_tokens: Some(reads), + total_tokens: Some(reads), + ..UsageCounts::default() + }; + summary + }; + store + .upsert_summary(&backfill("starts-inside", at(3, 0, 0), 1_000)) + .expect("summary"); + store + .upsert_summary(&backfill("starts-before", at(1, 0, 0), 50_000)) + .expect("summary"); + let query = CacheReportQuery::for_range(Some("2026-07-02"), Some("2026-07-09"), at(10, 0, 0)) + .expect("query"); + + let report = store.cache_report(&query, &Utc).expect("report"); + assert_eq!(report.totals.cache_read_tokens, 1_000); +} + +#[test] +fn missed_reuse_is_priced_as_rewritten_context_minus_cache_reads() { + let (store, source) = store_with_source(); + let model = |event: &mut UsageEvent, name: &str| { + event.model = Some(statsai_core::ModelInfo { + normalized_name: Some(name.to_string()), + ..statsai_core::ModelInfo::default() + }); + }; + let mut events = vec![ + call(&source, 1, at(1, 9, 0), 10, 0, 100_000), + // An hour and more later the context is written again, with the + // one-hour lifetime: 100,000 tokens at twice the input rate. + call(&source, 2, at(1, 10, 10), 10, 0, 100_000), + ]; + events[1].usage.cache_creation_1h_tokens = Some(100_000); + for event in &mut events { + model(event, "claude-sonnet-4-5"); + } + store.insert_events(&events).expect("insert"); + let query = CacheReportQuery::for_range(Some("2026-07-01"), Some("2026-07-01"), at(2, 0, 0)) + .expect("query"); + + let diagnostics = store + .cache_report(&query, &Utc) + .expect("report") + .diagnostics; + assert_eq!(diagnostics.missed_tokens, 100_000); + // $6.00 per million written for an hour, less $0.30 per million read. + assert_eq!(diagnostics.missed_cost_micro_usd, 570_000); + assert_eq!(diagnostics.unpriced_losses, 0); + assert_eq!( + health_by_day(&store)["2026-07-01"].missed_cost_micro_usd, + 570_000 + ); + + // A model without known pricing is counted, not priced. + let (store, source) = store_with_source(); + let mut events = vec![ + call(&source, 1, at(1, 9, 0), 10, 0, 100_000), + call(&source, 2, at(1, 10, 10), 10, 0, 100_000), + ]; + for event in &mut events { + model(event, "unpriced-model"); + } + store.insert_events(&events).expect("insert"); + let diagnostics = store + .cache_report(&query, &Utc) + .expect("report") + .diagnostics; + assert_eq!( + ( + diagnostics.losses, + diagnostics.unpriced_losses, + diagnostics.missed_cost_micro_usd + ), + (1, 1, 0) + ); +} + +#[test] +fn an_event_sharing_a_session_with_another_provider_is_counted_once() { + let (store, source) = store_with_source(); + let other_source = SourceLocation::local_adapter( + "opencode", + "test", + "0", + Path::new("/fixture/opencode"), + LocationOrigin::Configured, + ); + store.upsert_source(&other_source).expect("source"); + let mut other = test_store_event(&other_source, at(2, 9, 0), "other"); + other.provider = "opencode".to_string(); + other.session.session_id = "session_cache".to_string(); + other.usage.requests = Some(1); + store + .insert_events(&[call(&source, 1, at(2, 8, 0), 10, 0, 50_000), other]) + .expect("insert"); + let query = CacheReportQuery::for_range(Some("2026-07-02"), Some("2026-07-02"), at(3, 0, 0)) + .expect("query"); + + let report = store.cache_report(&query, &Utc).expect("report"); + assert_eq!(report.totals.requests, 2); + assert_eq!( + report + .providers + .iter() + .map(|row| (row.provider.as_str(), row.totals.requests)) + .collect::>(), + [("claude_code", 1), ("opencode", 1)] + ); +} + +#[test] +fn missed_reuse_is_priced_per_request_and_by_the_calls_own_model() { + let codex_source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/fixture/codex"), + LocationOrigin::Configured, + ); + let codex = |index: u64, minutes: u32, input: u64, read: u64| { + let mut event = call(&codex_source, index, at(1, 9, minutes), input, read, 0); + event.provider = "codex".to_string(); + event.usage.cache_creation_tokens = None; + event.model = Some(statsai_core::ModelInfo { + normalized_name: Some("gpt-6-astra".to_string()), + provider_model_id: Some("gpt-6-astra".to_string()), + ..statsai_core::ModelInfo::default() + }); + event + }; + let query = CacheReportQuery::for_range(Some("2026-07-01"), Some("2026-07-01"), at(2, 0, 0)) + .expect("query"); + + // A prompt over 272k tokens is priced at the long-context rates: $20 per + // million of input and $2 per million of cached input, not $10 and $1. + let store = Store::in_memory().expect("store"); + store.upsert_source(&codex_source).expect("source"); + store + .insert_events(&[codex(1, 0, 10, 300_000), codex(2, 1, 300_010, 0)]) + .expect("insert"); + let diagnostics = store + .cache_report(&query, &Utc) + .expect("report") + .diagnostics; + assert_eq!(diagnostics.missed_tokens, 300_000); + assert_eq!(diagnostics.missed_cost_micro_usd, 5_400_000); + + // A turn's call on a model without pricing stays unpriced, even though + // the turn's own model is priced. + let store = Store::in_memory().expect("store"); + store.upsert_source(&codex_source).expect("source"); + let mut turn = codex(1, 0, 50_010, 50_000); + turn.usage.requests = Some(2); + let model_call = |minute: i64, input: u64, read: u64, model: &str| ModelCall { + completed_at: at(1, 9, 0) + chrono::Duration::minutes(minute), + requested_at: None, + input_tokens: Some(input), + cache_read_tokens: Some(read), + cache_creation_tokens: None, + after_compaction: false, + model: Some(model.to_string()), + }; + turn.context = Some(CallContext { + calls: vec![ + model_call(0, 10, 50_000, "unpriced-model"), + model_call(1, 50_000, 0, "unpriced-model"), + ], + ..CallContext::default() + }); + store.insert_events(&[turn]).expect("insert"); + let diagnostics = store + .cache_report(&query, &Utc) + .expect("report") + .diagnostics; + assert_eq!( + ( + diagnostics.losses, + diagnostics.unpriced_losses, + diagnostics.missed_cost_micro_usd + ), + (1, 1, 0) + ); +} diff --git a/crates/statsai-store/src/tests/events.rs b/crates/statsai-store/src/tests/events.rs index eb74fc9..0fe92f0 100644 --- a/crates/statsai-store/src/tests/events.rs +++ b/crates/statsai-store/src/tests/events.rs @@ -68,6 +68,7 @@ fn inserts_events_idempotently() { }, created_at: now, imported_at: now, + context: None, }; assert!(store.insert_event(&event).expect("insert")); diff --git a/crates/statsai-store/src/tests/mod.rs b/crates/statsai-store/src/tests/mod.rs index d1bf1af..4a4c1f5 100644 --- a/crates/statsai-store/src/tests/mod.rs +++ b/crates/statsai-store/src/tests/mod.rs @@ -1,6 +1,7 @@ pub(crate) use super::*; mod accounts; +mod cache; mod dedupe; mod events; mod open; diff --git a/crates/statsai-store/src/tests/rollups/build.rs b/crates/statsai-store/src/tests/rollups/build.rs index 8e58c2d..bdb893d 100644 --- a/crates/statsai-store/src/tests/rollups/build.rs +++ b/crates/statsai-store/src/tests/rollups/build.rs @@ -30,7 +30,7 @@ fn sync_rollup_counts_underlying_requests_in_codex_turn_events() { ..ModelInfo::default() }); - let summary = build_sync_rollup_summary(&[first, second, legacy]); + let summary = build_sync_rollup_summary(&[first, second, legacy], None); assert_eq!(summary.usage.requests, Some(9)); assert_eq!(summary.models.len(), 2); @@ -188,7 +188,7 @@ fn sync_rollup_sums_micro_usd_before_rounding_to_cents() { event.cost.set_estimated_micro_usd(2_250); let events = vec![event; 1_000]; - let summary = build_sync_rollup_summary(&events); + let summary = build_sync_rollup_summary(&events, None); assert_eq!( summary.cost.estimated_api_equivalent_micro_usd, diff --git a/crates/statsai-store/src/tests/support.rs b/crates/statsai-store/src/tests/support.rs index 0f6fc58..9baee87 100644 --- a/crates/statsai-store/src/tests/support.rs +++ b/crates/statsai-store/src/tests/support.rs @@ -77,6 +77,7 @@ pub(super) fn test_store_event( }, created_at: now, imported_at: now, + context: None, } } diff --git a/crates/statsai-store/src/tests/sync_state/pending.rs b/crates/statsai-store/src/tests/sync_state/pending.rs index 9bc0337..2be2a41 100644 --- a/crates/statsai-store/src/tests/sync_state/pending.rs +++ b/crates/statsai-store/src/tests/sync_state/pending.rs @@ -120,6 +120,9 @@ fn pending_http_sync_summary_counts_include_retirement_only_reconciliation() { ); store.upsert_source(&source).expect("source"); let target = "https://api.example.com/api/sync/batches"; + store + .record_sync_target_cache_health_support("http", target, true) + .expect("receiver support"); let event = test_store_event(&source, Utc::now(), "retired-event"); store.insert_event(&event).expect("event"); let rollups = store.all_sync_rollup_summaries().expect("initial rollups"); diff --git a/crates/statsai-store/tests/cache_health_fixture.rs b/crates/statsai-store/tests/cache_health_fixture.rs new file mode 100644 index 0000000..03f4c81 --- /dev/null +++ b/crates/statsai-store/tests/cache_health_fixture.rs @@ -0,0 +1,304 @@ +//! The shared prompt-cache fixture. +//! +//! Synthetic Claude Code, Codex, and Cursor usage goes through the store; the +//! daily summaries it syncs and the totals `statsai report cache` prints are +//! written to `fixtures/cache_health.json`. Receivers of the summaries test +//! against a copy of that file, so the CLI report, the synced daily objects, +//! and anything aggregating them agree on one set of numbers. +//! +//! Set `STATSAI_UPDATE_FIXTURES=1` to rewrite the file after an intended change. + +use chrono::{DateTime, TimeZone, Utc}; +use serde_json::json; +use statsai_core::{ + sanitize_summary_for_sync, CallContext, Confidence, CostInfo, EventId, EventSource, + IdentitySource, LocationOrigin, ModelCall, ModelInfo, ParseEvidence, PrivacyInfo, PrivacyMode, + SessionInfo, SourceId, SourceKind, UsageCounts, UsageEvent, USAGE_EVENT_SCHEMA_VERSION, +}; +use statsai_store::{CacheReportQuery, Store}; +use std::path::Path; + +const FIXTURE: &str = "tests/fixtures/cache_health.json"; + +fn at(day: u32, hour: u32, minute: u32, second: u32) -> DateTime { + Utc.with_ymd_and_hms(2026, 7, day, hour, minute, second) + .single() + .expect("time") +} + +struct Call { + provider: &'static str, + session: &'static str, + file: &'static str, + line: u64, + at: DateTime, + input: u64, + read: u64, + write: Option, +} + +impl Call { + fn event(&self) -> UsageEvent { + let id = format!("{}_{}_{:03}", self.provider, self.session, self.line); + UsageEvent { + schema_version: USAGE_EVENT_SCHEMA_VERSION.to_string(), + event_id: EventId(format!("event_{id}")), + device_id: "device_fixture".to_string(), + provider: self.provider.to_string(), + source_id: SourceId(format!("src_fixture_{}", self.provider)), + provider_account_id: None, + subscription_id: None, + source: EventSource { + adapter_id: "fixture".to_string(), + adapter_version: "0".to_string(), + source_kind: SourceKind::LocalAdapter, + location_origin: Some(LocationOrigin::Configured), + source_type: "jsonl".to_string(), + source_path_hash: None, + source_record_id: None, + parse_confidence: Confidence::High, + }, + session: SessionInfo { + session_id: self.session.to_string(), + local_session_id_hash: None, + title: None, + started_at: self.at, + ended_at: None, + duration_seconds: None, + turn_started_at: None, + }, + model: Some(ModelInfo { + // Priced models, so missed reuse has a cost to check. + normalized_name: Some( + match self.provider { + "codex" => "gpt-6-sol", + "claude_code" => "claude-sonnet-4-5", + _ => "cursor-model", + } + .to_string(), + ), + ..ModelInfo::default() + }), + usage: UsageCounts { + input_tokens: Some(self.input), + output_tokens: Some(100), + cache_creation_tokens: self.write, + cache_read_tokens: Some(self.read), + requests: Some(1), + ..UsageCounts::default() + }, + runtime: None, + cost: CostInfo { + currency: "USD".to_string(), + estimated_api_equivalent_usd: None, + provider_reported_usd: None, + estimated_api_equivalent_micro_usd: None, + provider_reported_micro_usd: None, + pricing_source: None, + pricing_version: None, + confidence: Confidence::Low, + }, + parse_evidence: Some(ParseEvidence { + event_key_version: "fixture".to_string(), + source_file_path_hash: Some(self.file.to_string()), + source_line_number: Some(self.line), + source_record_id: None, + model_inferred: false, + timestamp_inferred: false, + account_identity_source: IdentitySource::Unresolved, + }), + project: None, + git: None, + privacy: PrivacyInfo { + mode: PrivacyMode::MetadataOnly, + contains_prompt_text: false, + contains_response_text: false, + contains_file_paths: false, + }, + created_at: self.at, + imported_at: self.at, + context: None, + } + } +} + +fn claude(line: u64, at: DateTime, input: u64, read: u64, write: u64) -> UsageEvent { + Call { + provider: "claude_code", + session: "session_fixture_claude", + file: "claude-main", + line, + at, + input, + read, + write: Some(write), + } + .event() +} + +fn events() -> Vec { + let mut events = vec![ + // Day one: a cold start, growth, a short-gap partial loss, a pause. + claude(1, at(1, 9, 0, 0), 20, 0, 40_000), + claude(2, at(1, 9, 0, 30), 20, 40_000, 2_000), + claude(3, at(1, 9, 1, 10), 20, 15_000, 27_500), + claude(4, at(1, 9, 3, 0), 20, 42_500, 1_000), + claude(5, at(1, 9, 9, 0), 20, 43_500, 900), + // Across midnight after a long pause: a full loss. + claude(6, at(1, 23, 50, 0), 20, 44_400, 600), + claude(7, at(2, 1, 0, 0), 20, 0, 45_200), + claude(8, at(2, 1, 0, 40), 20, 45_200, 800), + // A request a router served without the cache: neither read nor written. + claude(11, at(2, 1, 2, 0), 46_020, 0, 0), + ]; + // A compaction rewrites the prompt: a restart, not a loss. + let mut compacted = claude(9, at(2, 1, 5, 0), 20, 0, 9_000); + compacted.context = Some(CallContext { + after_compaction: true, + ..CallContext::default() + }); + events.push(compacted); + // A sub-agent in the same session is its own stream. + let mut agent = claude(10, at(1, 9, 2, 0), 20, 0, 8_000); + agent.context = Some(CallContext { + agent_hash: Some("fixture_agent".to_string()), + ..CallContext::default() + }); + events.push(agent); + + // A Codex turn listing its calls, with recorded request starts. + let mut turn = Call { + provider: "codex", + session: "session_fixture_codex", + file: "codex-rollout", + line: 1, + at: at(2, 10, 0, 0), + input: 0, + read: 0, + write: Some(0), + } + .event(); + let calls = [ + (at(2, 10, 0, 10), at(2, 10, 0, 0), 30_000, 0), + (at(2, 10, 0, 30), at(2, 10, 0, 12), 3_000, 29_952), + (at(2, 10, 7, 0), at(2, 10, 6, 40), 26_000, 7_168), + ]; + turn.usage.requests = Some(calls.len() as u64); + turn.usage.input_tokens = Some(calls.iter().map(|call| call.2).sum()); + turn.usage.cache_read_tokens = Some(calls.iter().map(|call| call.3).sum()); + turn.created_at = at(2, 10, 7, 30); + turn.context = Some(CallContext { + calls: calls + .iter() + .map(|(completed_at, requested_at, input, read)| ModelCall { + completed_at: *completed_at, + requested_at: Some(*requested_at), + input_tokens: Some(*input), + cache_read_tokens: Some(*read), + cache_creation_tokens: Some(0), + after_compaction: false, + model: None, + }) + .collect(), + ..CallContext::default() + }); + events.push(turn); + // An older Codex turn without a per-call breakdown and without write + // telemetry: counted, not classified. + let mut legacy = Call { + provider: "codex", + session: "session_fixture_codex", + file: "codex-rollout", + line: 2, + at: at(2, 11, 0, 0), + input: 4_000, + read: 60_000, + write: None, + } + .event(); + legacy.usage.requests = Some(2); + events.push(legacy); + + // Cursor has no call order: token totals only. + events.push( + Call { + provider: "cursor", + session: "session_fixture_cursor", + file: "cursor-export", + line: 1, + at: at(2, 12, 0, 0), + input: 1_000, + read: 9_000, + write: Some(500), + } + .event(), + ); + events +} + +fn fixture() -> serde_json::Value { + let store = Store::in_memory().expect("store"); + store.insert_events(&events()).expect("insert"); + let mut summaries = store + .all_sync_rollup_summaries() + .expect("summaries") + .into_iter() + .map(sanitize_summary_for_sync) + .collect::>(); + summaries.sort_by(|left, right| left.summary_id.0.cmp(&right.summary_id.0)); + let report = store + .cache_report( + &CacheReportQuery::for_range(Some("2026-07-01"), Some("2026-07-02"), at(3, 0, 0, 0)) + .expect("query"), + &Utc, + ) + .expect("report"); + json!({ + "summaries": summaries, + "report": { + "totals": report.totals, + "diagnostics": report.diagnostics, + "gap_histogram": report.gap_histogram, + "days": report.days.iter().map(|day| json!({ + "day": day.day, + "losses": day.diagnostics.as_ref().map_or(0, |health| health.losses), + "missed": day.diagnostics.as_ref().map_or(0, |health| health.missed_tokens), + "missed_cost_micro_usd": day + .diagnostics + .as_ref() + .map_or(0, |health| health.missed_cost_micro_usd), + })).collect::>(), + }, + }) +} + +#[test] +fn the_shared_cache_health_fixture_is_current() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join(FIXTURE); + let current = serde_json::to_string_pretty(&fixture()).expect("json") + "\n"; + if std::env::var_os("STATSAI_UPDATE_FIXTURES").is_some() { + std::fs::write(&path, ¤t).expect("write fixture"); + return; + } + let stored = std::fs::read_to_string(&path).unwrap_or_default(); + assert!( + stored == current, + "{FIXTURE} is out of date; rerun with STATSAI_UPDATE_FIXTURES=1 and copy it to its receivers" + ); +} + +#[test] +fn the_fixture_report_matches_its_daily_summaries() { + let fixture = fixture(); + let mut merged = statsai_core::CacheHealthV1::default(); + for summary in fixture["summaries"].as_array().expect("summaries") { + if let Some(health) = summary.pointer("/metrics/cache_health") { + merged.merge(&serde_json::from_value(health.clone()).expect("cache health")); + } + } + assert_eq!( + serde_json::to_value(&merged).expect("json"), + fixture["report"]["diagnostics"] + ); + assert_eq!((merged.unclassifiable, merged.boundaries), (2, 1)); +} diff --git a/crates/statsai-store/tests/fixtures/cache_health.json b/crates/statsai-store/tests/fixtures/cache_health.json new file mode 100644 index 0000000..70047ba --- /dev/null +++ b/crates/statsai-store/tests/fixtures/cache_health.json @@ -0,0 +1,785 @@ +{ + "report": { + "days": [ + { + "day": "2026-07-01", + "losses": 1, + "missed": 27000, + "missed_cost_micro_usd": 93150 + }, + { + "day": "2026-07-02", + "losses": 2, + "missed": 67784, + "missed_cost_micro_usd": 196262 + } + ], + "diagnostics": { + "below_threshold": 1, + "boundaries": 1, + "comparable": 8, + "first_cold": 3, + "first_observed": 3, + "full_losses": 1, + "gap_comparable": [ + 3, + 1, + 2, + 0, + 0, + 0, + 0, + 2 + ], + "gap_losses": [ + 1, + 0, + 1, + 0, + 0, + 0, + 0, + 1 + ], + "gap_missed_tokens": [ + 27000, + 0, + 22784, + 0, + 0, + 0, + 0, + 45000 + ], + "losses": 3, + "missed_cost_micro_usd": 289412, + "missed_tokens": 94784, + "requests": 16, + "timing_estimated": 7, + "uncached": 1, + "unclassifiable": 2, + "version": 1, + "write_split_unavailable_input_tokens": 4000 + }, + "gap_histogram": [ + { + "comparable": 3, + "label": "<1m", + "losses": 1, + "max_minutes": 1, + "min_minutes": 0, + "missed_tokens": 27000 + }, + { + "comparable": 1, + "label": "1–5m", + "losses": 0, + "max_minutes": 5, + "min_minutes": 1, + "missed_tokens": 0 + }, + { + "comparable": 2, + "label": "5–10m", + "losses": 1, + "max_minutes": 10, + "min_minutes": 5, + "missed_tokens": 22784 + }, + { + "comparable": 0, + "label": "10–15m", + "losses": 0, + "max_minutes": 15, + "min_minutes": 10, + "missed_tokens": 0 + }, + { + "comparable": 0, + "label": "15–30m", + "losses": 0, + "max_minutes": 30, + "min_minutes": 15, + "missed_tokens": 0 + }, + { + "comparable": 0, + "label": "30–45m", + "losses": 0, + "max_minutes": 45, + "min_minutes": 30, + "missed_tokens": 0 + }, + { + "comparable": 0, + "label": "45–60m", + "losses": 0, + "max_minutes": 60, + "min_minutes": 45, + "missed_tokens": 0 + }, + { + "comparable": 2, + "label": "60m+", + "losses": 1, + "max_minutes": null, + "min_minutes": 60, + "missed_tokens": 45000 + } + ], + "totals": { + "avg_logical_input_per_request": 34261.17647058824, + "cache_creation_tokens": 135500, + "cache_hit_ratio": 0.578119634640478, + "cache_read_tokens": 336720, + "input_tokens": 110220, + "logical_input_tokens": 582440, + "requests": 17 + } + }, + "summaries": [ + { + "cost": { + "confidence": "medium", + "currency": "USD", + "estimated_api_equivalent_usd": null, + "pricing_source": "local_rollup", + "pricing_version": null, + "provider_reported_usd": null + }, + "device_id": "device_fixture", + "imported_at": "2026-07-02T11:00:00Z", + "metadata": { + "last_computed_at": "2026-07-02T11:00:00Z", + "summary_format": "daily_rollup.v1", + "summary_version": "14", + "total_messages": null, + "total_sessions": 1 + }, + "metrics": { + "active_seconds": null, + "assistant_messages": null, + "cache_health": { + "below_threshold": 1, + "boundaries": 0, + "comparable": 1, + "first_cold": 1, + "first_observed": 1, + "full_losses": 0, + "gap_comparable": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "gap_losses": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "gap_missed_tokens": [ + 0, + 0, + 22784, + 0, + 0, + 0, + 0, + 0 + ], + "losses": 1, + "missed_cost_micro_usd": 41012, + "missed_tokens": 22784, + "requests": 5, + "timing_estimated": 0, + "unclassifiable": 2, + "version": 1, + "write_split_unavailable_input_tokens": 4000 + }, + "cache_hit_ratio": { + "avg": 0.6618419683728672, + "max": 0.9375, + "min": 0.3861839367457345, + "p50": 0.3861839367457345, + "p95": 0.9375, + "samples": 2, + "sum": 1.3236839367457345 + }, + "developer_messages": null, + "generated_tps": null, + "latency_ms": null, + "overall_generated_tps": null, + "overall_visible_tps": null, + "reasoning_share": { + "avg": 0.0, + "max": 0.0, + "min": 0.0, + "p50": 0.0, + "p95": 0.0, + "samples": 2, + "sum": 0.0 + }, + "time_to_first_token_ms": null, + "total_messages": null, + "tracked_output_tokens": null, + "tracked_reasoning_tokens": null, + "tracked_requests": null, + "user_messages": null, + "visible_tps": null + }, + "model": null, + "models": [ + { + "cost": { + "confidence": "medium", + "currency": "USD", + "estimated_api_equivalent_usd": null, + "pricing_source": "local_rollup", + "pricing_version": null, + "provider_reported_usd": null + }, + "model": { + "name": null, + "normalized_name": "gpt-6-sol", + "provider_model_id": null, + "reasoning_level": null, + "reasoning_level_raw": null, + "speed": null + }, + "usage": { + "cache_creation_1h_tokens": 0, + "cache_creation_5m_tokens": 0, + "cache_creation_tokens": 0, + "cache_read_tokens": 97120, + "input_tokens": 63000, + "local_eval_tokens": null, + "local_prompt_eval_tokens": null, + "output_tokens": 200, + "reasoning_tokens": 0, + "requests": 5, + "total_tokens": 160320 + } + } + ], + "observed_at": "2026-07-02T11:00:00Z", + "parse_evidence": null, + "period_end": "2026-07-03T00:00:00Z", + "period_start": "2026-07-02T00:00:00Z", + "privacy": { + "contains_file_paths": false, + "contains_prompt_text": false, + "contains_response_text": false, + "mode": "metadata_only" + }, + "project": null, + "provider": "codex", + "provider_account_id": null, + "schema_version": "usage_summary.v1", + "source": { + "adapter_id": "fixture", + "adapter_version": "0", + "location_origin": "configured", + "parse_confidence": "high", + "source_kind": "local_adapter", + "source_path_hash": null, + "source_record_id": null, + "source_type": "jsonl" + }, + "source_id": "src_fixture_codex", + "summary_id": "sum_547c6aa40f5895680f0045854f4a95e3", + "usage": { + "cache_creation_1h_tokens": 0, + "cache_creation_5m_tokens": 0, + "cache_creation_tokens": 0, + "cache_read_tokens": 97120, + "input_tokens": 63000, + "local_eval_tokens": null, + "local_prompt_eval_tokens": null, + "output_tokens": 200, + "reasoning_tokens": 0, + "requests": 5, + "total_tokens": 160320 + } + }, + { + "cost": { + "confidence": "medium", + "currency": "USD", + "estimated_api_equivalent_usd": null, + "pricing_source": "local_rollup", + "pricing_version": null, + "provider_reported_usd": null + }, + "device_id": "device_fixture", + "imported_at": "2026-07-02T12:00:00Z", + "metadata": { + "last_computed_at": "2026-07-02T12:00:00Z", + "summary_format": "daily_rollup.v1", + "summary_version": "14", + "total_messages": null, + "total_sessions": 1 + }, + "metrics": { + "active_seconds": null, + "assistant_messages": null, + "cache_hit_ratio": { + "avg": 0.8571428571428571, + "max": 0.8571428571428571, + "min": 0.8571428571428571, + "p50": 0.8571428571428571, + "p95": 0.8571428571428571, + "samples": 1, + "sum": 0.8571428571428571 + }, + "developer_messages": null, + "generated_tps": null, + "latency_ms": null, + "overall_generated_tps": null, + "overall_visible_tps": null, + "reasoning_share": { + "avg": 0.0, + "max": 0.0, + "min": 0.0, + "p50": 0.0, + "p95": 0.0, + "samples": 1, + "sum": 0.0 + }, + "time_to_first_token_ms": null, + "total_messages": null, + "tracked_output_tokens": null, + "tracked_reasoning_tokens": null, + "tracked_requests": null, + "user_messages": null, + "visible_tps": null + }, + "model": null, + "models": [ + { + "cost": { + "confidence": "medium", + "currency": "USD", + "estimated_api_equivalent_usd": null, + "pricing_source": "local_rollup", + "pricing_version": null, + "provider_reported_usd": null + }, + "model": { + "name": null, + "normalized_name": "cursor-model", + "provider_model_id": null, + "reasoning_level": null, + "reasoning_level_raw": null, + "speed": null + }, + "usage": { + "cache_creation_1h_tokens": 0, + "cache_creation_5m_tokens": 0, + "cache_creation_tokens": 500, + "cache_read_tokens": 9000, + "input_tokens": 1000, + "local_eval_tokens": null, + "local_prompt_eval_tokens": null, + "output_tokens": 100, + "reasoning_tokens": 0, + "requests": 1, + "total_tokens": 10600 + } + } + ], + "observed_at": "2026-07-02T12:00:00Z", + "parse_evidence": null, + "period_end": "2026-07-03T00:00:00Z", + "period_start": "2026-07-02T00:00:00Z", + "privacy": { + "contains_file_paths": false, + "contains_prompt_text": false, + "contains_response_text": false, + "mode": "metadata_only" + }, + "project": null, + "provider": "cursor", + "provider_account_id": null, + "schema_version": "usage_summary.v1", + "source": { + "adapter_id": "fixture", + "adapter_version": "0", + "location_origin": "configured", + "parse_confidence": "high", + "source_kind": "local_adapter", + "source_path_hash": null, + "source_record_id": null, + "source_type": "jsonl" + }, + "source_id": "src_fixture_cursor", + "summary_id": "sum_8e4eef2e63c1cf9e4c9c0caa09b6f554", + "usage": { + "cache_creation_1h_tokens": 0, + "cache_creation_5m_tokens": 0, + "cache_creation_tokens": 500, + "cache_read_tokens": 9000, + "input_tokens": 1000, + "local_eval_tokens": null, + "local_prompt_eval_tokens": null, + "output_tokens": 100, + "reasoning_tokens": 0, + "requests": 1, + "total_tokens": 10600 + } + }, + { + "cost": { + "confidence": "medium", + "currency": "USD", + "estimated_api_equivalent_usd": null, + "pricing_source": "local_rollup", + "pricing_version": null, + "provider_reported_usd": null + }, + "device_id": "device_fixture", + "imported_at": "2026-07-02T01:05:00Z", + "metadata": { + "last_computed_at": "2026-07-02T01:05:00Z", + "summary_format": "daily_rollup.v1", + "summary_version": "14", + "total_messages": null, + "total_sessions": 1 + }, + "metrics": { + "active_seconds": null, + "assistant_messages": null, + "cache_health": { + "below_threshold": 0, + "boundaries": 1, + "comparable": 2, + "first_cold": 0, + "first_observed": 0, + "full_losses": 1, + "gap_comparable": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1 + ], + "gap_losses": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1 + ], + "gap_missed_tokens": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 45000 + ], + "losses": 1, + "missed_cost_micro_usd": 155250, + "missed_tokens": 45000, + "requests": 4, + "timing_estimated": 2, + "uncached": 1, + "unclassifiable": 0, + "version": 1, + "write_split_unavailable_input_tokens": 0 + }, + "cache_hit_ratio": { + "avg": 0.24554541503694047, + "max": 0.982181660147762, + "min": 0.0, + "p50": 0.0, + "p95": 0.982181660147762, + "samples": 4, + "sum": 0.982181660147762 + }, + "developer_messages": null, + "generated_tps": null, + "latency_ms": null, + "overall_generated_tps": null, + "overall_visible_tps": null, + "reasoning_share": { + "avg": 0.0, + "max": 0.0, + "min": 0.0, + "p50": 0.0, + "p95": 0.0, + "samples": 4, + "sum": 0.0 + }, + "time_to_first_token_ms": null, + "total_messages": null, + "tracked_output_tokens": null, + "tracked_reasoning_tokens": null, + "tracked_requests": null, + "user_messages": null, + "visible_tps": null + }, + "model": null, + "models": [ + { + "cost": { + "confidence": "medium", + "currency": "USD", + "estimated_api_equivalent_usd": null, + "pricing_source": "local_rollup", + "pricing_version": null, + "provider_reported_usd": null + }, + "model": { + "name": null, + "normalized_name": "claude-sonnet-4-5", + "provider_model_id": null, + "reasoning_level": null, + "reasoning_level_raw": null, + "speed": null + }, + "usage": { + "cache_creation_1h_tokens": 0, + "cache_creation_5m_tokens": 0, + "cache_creation_tokens": 55000, + "cache_read_tokens": 45200, + "input_tokens": 46080, + "local_eval_tokens": null, + "local_prompt_eval_tokens": null, + "output_tokens": 400, + "reasoning_tokens": 0, + "requests": 4, + "total_tokens": 146680 + } + } + ], + "observed_at": "2026-07-02T01:05:00Z", + "parse_evidence": null, + "period_end": "2026-07-03T00:00:00Z", + "period_start": "2026-07-02T00:00:00Z", + "privacy": { + "contains_file_paths": false, + "contains_prompt_text": false, + "contains_response_text": false, + "mode": "metadata_only" + }, + "project": null, + "provider": "claude_code", + "provider_account_id": null, + "schema_version": "usage_summary.v1", + "source": { + "adapter_id": "fixture", + "adapter_version": "0", + "location_origin": "configured", + "parse_confidence": "high", + "source_kind": "local_adapter", + "source_path_hash": null, + "source_record_id": null, + "source_type": "jsonl" + }, + "source_id": "src_fixture_claude_code", + "summary_id": "sum_9ebe14eb7cbd9f700433ec92c14e824d", + "usage": { + "cache_creation_1h_tokens": 0, + "cache_creation_5m_tokens": 0, + "cache_creation_tokens": 55000, + "cache_read_tokens": 45200, + "input_tokens": 46080, + "local_eval_tokens": null, + "local_prompt_eval_tokens": null, + "output_tokens": 400, + "reasoning_tokens": 0, + "requests": 4, + "total_tokens": 146680 + } + }, + { + "cost": { + "confidence": "medium", + "currency": "USD", + "estimated_api_equivalent_usd": null, + "pricing_source": "local_rollup", + "pricing_version": null, + "provider_reported_usd": null + }, + "device_id": "device_fixture", + "imported_at": "2026-07-01T23:50:00Z", + "metadata": { + "last_computed_at": "2026-07-01T23:50:00Z", + "summary_format": "daily_rollup.v1", + "summary_version": "14", + "total_messages": null, + "total_sessions": 1 + }, + "metrics": { + "active_seconds": null, + "assistant_messages": null, + "cache_health": { + "below_threshold": 0, + "boundaries": 0, + "comparable": 5, + "first_cold": 2, + "first_observed": 2, + "full_losses": 0, + "gap_comparable": [ + 2, + 1, + 1, + 0, + 0, + 0, + 0, + 1 + ], + "gap_losses": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "gap_missed_tokens": [ + 27000, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "losses": 1, + "missed_cost_micro_usd": 93150, + "missed_tokens": 27000, + "requests": 7, + "timing_estimated": 5, + "unclassifiable": 0, + "version": 1, + "write_split_unavailable_input_tokens": 0 + }, + "cache_hit_ratio": { + "avg": 0.6066831814028905, + "max": 0.986228342958685, + "min": 0.0, + "p50": 0.951927653498334, + "p95": 0.986228342958685, + "samples": 7, + "sum": 4.246782269820233 + }, + "developer_messages": null, + "generated_tps": null, + "latency_ms": null, + "overall_generated_tps": null, + "overall_visible_tps": null, + "reasoning_share": { + "avg": 0.0, + "max": 0.0, + "min": 0.0, + "p50": 0.0, + "p95": 0.0, + "samples": 7, + "sum": 0.0 + }, + "time_to_first_token_ms": null, + "total_messages": null, + "tracked_output_tokens": null, + "tracked_reasoning_tokens": null, + "tracked_requests": null, + "user_messages": null, + "visible_tps": null + }, + "model": null, + "models": [ + { + "cost": { + "confidence": "medium", + "currency": "USD", + "estimated_api_equivalent_usd": null, + "pricing_source": "local_rollup", + "pricing_version": null, + "provider_reported_usd": null + }, + "model": { + "name": null, + "normalized_name": "claude-sonnet-4-5", + "provider_model_id": null, + "reasoning_level": null, + "reasoning_level_raw": null, + "speed": null + }, + "usage": { + "cache_creation_1h_tokens": 0, + "cache_creation_5m_tokens": 0, + "cache_creation_tokens": 80000, + "cache_read_tokens": 185400, + "input_tokens": 140, + "local_eval_tokens": null, + "local_prompt_eval_tokens": null, + "output_tokens": 700, + "reasoning_tokens": 0, + "requests": 7, + "total_tokens": 266240 + } + } + ], + "observed_at": "2026-07-01T23:50:00Z", + "parse_evidence": null, + "period_end": "2026-07-02T00:00:00Z", + "period_start": "2026-07-01T00:00:00Z", + "privacy": { + "contains_file_paths": false, + "contains_prompt_text": false, + "contains_response_text": false, + "mode": "metadata_only" + }, + "project": null, + "provider": "claude_code", + "provider_account_id": null, + "schema_version": "usage_summary.v1", + "source": { + "adapter_id": "fixture", + "adapter_version": "0", + "location_origin": "configured", + "parse_confidence": "high", + "source_kind": "local_adapter", + "source_path_hash": null, + "source_record_id": null, + "source_type": "jsonl" + }, + "source_id": "src_fixture_claude_code", + "summary_id": "sum_bf315597fd8d01330eb4d20bcff5b1ab", + "usage": { + "cache_creation_1h_tokens": 0, + "cache_creation_5m_tokens": 0, + "cache_creation_tokens": 80000, + "cache_read_tokens": 185400, + "input_tokens": 140, + "local_eval_tokens": null, + "local_prompt_eval_tokens": null, + "output_tokens": 700, + "reasoning_tokens": 0, + "requests": 7, + "total_tokens": 266240 + } + } + ] +} diff --git a/crates/statsai-sync/src/lib.rs b/crates/statsai-sync/src/lib.rs index 6fd5a92..3e54589 100644 --- a/crates/statsai-sync/src/lib.rs +++ b/crates/statsai-sync/src/lib.rs @@ -857,6 +857,7 @@ mod tests { }, created_at: Utc::now(), imported_at: Utc::now(), + context: None, } } diff --git a/crates/statsai/src/cli/args/mod.rs b/crates/statsai/src/cli/args/mod.rs index 0d80179..b472a16 100644 --- a/crates/statsai/src/cli/args/mod.rs +++ b/crates/statsai/src/cli/args/mod.rs @@ -280,6 +280,8 @@ pub(crate) enum SchemaSubcommand { SyncBatch, #[command(about = "Print the quota_window_sync_projection.v1 JSON Schema")] QuotaWindowProjection, + #[command(about = "Print the cache_report.v1 JSON Schema")] + CacheReport, } #[derive(Debug, Args)] diff --git a/crates/statsai/src/cli/args/report.rs b/crates/statsai/src/cli/args/report.rs index 55adce2..8dae40f 100644 --- a/crates/statsai/src/cli/args/report.rs +++ b/crates/statsai/src/cli/args/report.rs @@ -56,4 +56,31 @@ pub(crate) enum ReportSubcommand { #[arg(long, help = "Include subscription-value rows")] subscriptions: bool, }, + #[command(about = "Show prompt-cache reuse, suspected cache losses, and context per call")] + Cache(CacheReportArgs), +} + +#[derive(Debug, Args)] +pub(crate) struct CacheReportArgs { + #[arg( + long, + help = "Range start (YYYY-MM-DD or RFC3339). Date-only values are UTC days. Defaults to the last 7 days" + )] + pub(crate) from: Option, + #[arg(long, help = "Range end (YYYY-MM-DD or RFC3339). Defaults to now")] + pub(crate) to: Option, + #[arg(long, conflicts_with_all = ["from", "to"], help = "Report all stored history")] + pub(crate) all: bool, + #[arg(long, help = "Provider filter")] + pub(crate) provider: Option, + #[arg(long, help = "Provider account id, or `unassigned`")] + pub(crate) account: Option, + #[arg(long, help = "Session id (session_…) or a prefix of one")] + pub(crate) session: Option, + #[arg(long, help = "Add local 10-minute buckets of calls")] + pub(crate) timeline: bool, + #[arg(long, help = "List every call with its verdict and evidence")] + pub(crate) details: bool, + #[arg(long, help = "Output as JSON")] + pub(crate) json: bool, } diff --git a/crates/statsai/src/cli/cache.rs b/crates/statsai/src/cli/cache.rs new file mode 100644 index 0000000..e409fa2 --- /dev/null +++ b/crates/statsai/src/cli/cache.rs @@ -0,0 +1,363 @@ +use anyhow::Result; +use chrono::{Local, Utc}; +use statsai_core::{ + micro_usd_to_cents_rounded, AnalyzedCacheCall, CacheBoundaryReason, CacheCallClass, + CacheGapTiming, CacheLoss, CacheReport, CacheTokenTotals, CacheUnclassifiableReason, + CacheWriteTtl, CACHE_LOSS_MIN_PERCENT, CACHE_LOSS_MIN_TOKENS, +}; +use statsai_store::{CacheReportQuery, Store}; + +use super::args::CacheReportArgs; +use super::format::{format_cost, format_u64, truncate_label}; +use super::source::canonical_provider; + +pub(crate) fn cache_report(args: CacheReportArgs, store: &Store) -> Result<()> { + let report = store.cache_report(&cache_report_query(&args, Utc::now())?, &Local)?; + if args.json { + println!("{}", serde_json::to_string_pretty(&report)?); + } else { + print_cache_report(&report); + } + Ok(()) +} + +pub(super) fn cache_report_query( + args: &CacheReportArgs, + now: chrono::DateTime, +) -> Result { + let mut query = if args.all { + CacheReportQuery::all_time(now) + } else { + CacheReportQuery::for_range(args.from.as_deref(), args.to.as_deref(), now)? + }; + query.provider = args + .provider + .as_deref() + .map(canonical_provider) + .transpose()?; + query.account = args.account.clone(); + query.session = args.session.clone(); + query.timeline = args.timeline; + query.details = args.details; + Ok(query) +} + +fn print_cache_report(report: &CacheReport) { + println!("statsai cache report: {}", report.label); + match report.since { + Some(since) => println!( + "range: {} to {}", + since.to_rfc3339(), + report.until.to_rfc3339() + ), + None => println!( + "range: all stored events through {}", + report.until.to_rfc3339() + ), + } + println!(); + print_totals(report); + println!(); + print_diagnostics(report); + if !report.providers.is_empty() { + println!(); + print_providers(report); + } + if let Some(timeline) = report.timeline.as_deref() { + println!(); + print_timeline(timeline); + } + if let Some(details) = report.details.as_deref() { + println!(); + print_details(details); + } +} + +fn print_totals(report: &CacheReport) { + let totals = &report.totals; + let unsplit = report.diagnostics.write_split_unavailable_input_tokens; + println!( + "{:<20}{} of {} logical input tokens", + "cache hit", + format_share(totals.cache_hit_ratio), + format_u64(totals.logical_input_tokens) + ); + let mut breakdown = format!( + "cache reads {} · cache writes {} · ordinary input {}", + format_u64(totals.cache_read_tokens), + format_u64(totals.cache_creation_tokens), + format_u64(totals.input_tokens.saturating_sub(unsplit)) + ); + if unsplit > 0 { + breakdown.push_str(&format!( + " · uncached input, writes not reported {}", + format_u64(unsplit) + )); + } + println!("{:<20}{breakdown}", "input breakdown"); + println!( + "{:<20}{} calls · {} input tokens per call", + "context processed", + format_u64(totals.requests), + format_average(totals) + ); +} + +fn format_average(totals: &CacheTokenTotals) -> String { + totals + .avg_logical_input_per_request + .map(|average| format_u64(average.round() as u64)) + .unwrap_or_else(|| "—".to_string()) +} + +fn print_diagnostics(report: &CacheReport) { + let health = &report.diagnostics; + println!("diagnostics (detector v{})", report.detector_version); + if health.requests == 0 { + println!(" no Claude Code or Codex calls in this range"); + return; + } + println!( + "{:<20}{} analyzed · {} unclassifiable · {} timing-estimated", + "coverage", + format_u64(health.analyzed()), + format_u64(health.unclassifiable), + format_u64(health.timing_estimated) + ); + println!( + "{:<20}{} streams · {} cold", + "first observed", + format_u64(health.first_observed), + format_u64(health.first_cold) + ); + let boundaries = &report.boundaries; + println!( + "{:<20}{} (model change {} · compaction {} · evidence gap {})", + "restarts", + format_u64(health.boundaries), + format_u64(boundaries.model_change), + format_u64(boundaries.compaction), + format_u64(boundaries.evidence_gap) + ); + println!( + "{:<20}{} · {} suspected losses ({} full · {} partial)", + "comparable calls", + format_u64(health.comparable), + format_u64(health.losses), + format_u64(health.full_losses), + format_u64(health.losses.saturating_sub(health.full_losses)) + ); + if health.losses > 0 { + let logical = report.totals.logical_input_tokens; + println!( + "{:<20}{} reusable tokens not read from the cache ({} of logical input)", + "missed reuse", + format_u64(health.missed_tokens), + format_share((logical > 0).then(|| health.missed_tokens as f64 / logical as f64)) + ); + let mut cost = format!( + "{} extra at API prices", + format_cost(Some(micro_usd_to_cents_rounded( + i64::try_from(health.missed_cost_micro_usd).unwrap_or(i64::MAX) + ))) + ); + if health.unpriced_losses > 0 { + cost.push_str(&format!( + ", leaving out {} losses on models without known pricing", + format_u64(health.unpriced_losses) + )); + } + println!("{:<20}{cost}", "missed reuse cost"); + } + if health.uncached > 0 { + println!( + "{:<20}{} calls neither read nor wrote the cache, so are not judged", + "cache not used", + format_u64(health.uncached) + ); + } + if health.below_threshold > 0 { + println!( + "{:<20}{} calls followed less than {} reusable tokens", + "too small to judge", + format_u64(health.below_threshold), + format_u64(CACHE_LOSS_MIN_TOKENS) + ); + } + println!(); + println!( + "{:<26} {:>12} {:>10} {:>10} {:>16}", + "gap since previous call", "comparable", "losses", "loss rate", "missed tokens" + ); + for row in &report.gap_histogram { + println!( + "{:<26} {:>12} {:>10} {:>10} {:>16}", + row.label, + format_u64(row.comparable), + format_u64(row.losses), + format_share((row.comparable > 0).then(|| row.losses as f64 / row.comparable as f64)), + format_u64(row.missed_tokens) + ); + } + println!(); + println!( + "A suspected loss is a call whose cached reads fell at least {}% and {} tokens below", + CACHE_LOSS_MIN_PERCENT, + format_u64(CACHE_LOSS_MIN_TOKENS) + ); + println!("the context the previous call left reusable. It is read from token counts, not"); + println!("confirmed by the provider. Timing-estimated gaps run between recorded responses"); + println!("and include the call's own run time."); +} + +fn print_providers(report: &CacheReport) { + println!( + "{:<14} {:>10} {:>8} {:>16} {:>12} {:>8}", + "provider", "calls", "hit", "input/call", "comparable", "losses" + ); + for row in &report.providers { + let health = row.diagnostics.as_ref(); + println!( + "{:<14} {:>10} {:>8} {:>16} {:>12} {:>8}", + truncate_label(&row.provider, 14), + format_u64(row.totals.requests), + format_share(row.totals.cache_hit_ratio), + format_average(&row.totals), + optional_count(health.map(|health| health.comparable)), + optional_count(health.map(|health| health.losses)), + ); + } +} + +fn print_timeline(timeline: &[statsai_core::CacheTimelineRow]) { + println!( + "{:<26} {:>8} {:>14} {:>8} {:>7} {:>9} {:>7}", + "local 10-minute bucket", "calls", "context", "hit", "first", "restarts", "losses" + ); + for row in timeline { + println!( + "{:<26} {:>8} {:>14} {:>8} {:>7} {:>9} {:>7}", + row.start, + format_u64(row.requests), + format_u64(row.logical_input_tokens), + format_share(row.cache_hit_ratio), + format_u64(row.first_observed), + format_u64(row.boundaries), + format_u64(row.losses) + ); + } +} + +fn print_details(details: &[AnalyzedCacheCall]) { + println!( + "{:<19} {:<12} {:<16} {:<18} {:>10} {:>10} {:>9} {:>10} {:>8} verdict", + "completed (local)", + "provider", + "session", + "model", + "input", + "read", + "write", + "baseline", + "gap" + ); + for call in details { + let agent = call + .agent_hash + .as_deref() + .map(|hash| format!("/{}", &hash[..hash.len().min(6)])) + .unwrap_or_default(); + println!( + "{:<19} {:<12} {:<16} {:<18} {:>10} {:>10} {:>9} {:>10} {:>8} {}", + call.completed_at + .with_timezone(&Local) + .format("%Y-%m-%d %H:%M:%S"), + truncate_label(&call.provider, 12), + truncate_label(&format!("{}{agent}", call.session_id), 16), + truncate_label(call.model.as_deref().unwrap_or("unknown"), 18), + format_u64(call.input_tokens), + format_u64(call.cache_read_tokens), + call.cache_creation_tokens + .map(format_u64) + .unwrap_or_else(|| "—".to_string()), + call.verdict + .baseline_tokens + .map(format_u64) + .unwrap_or_else(|| "—".to_string()), + format_gap(call), + verdict_label(call) + ); + } +} + +fn format_gap(call: &AnalyzedCacheCall) -> String { + let Some(seconds) = call.verdict.gap_seconds else { + return "—".to_string(); + }; + let text = if seconds >= 3600 { + format!("{}h{:02}m", seconds / 3600, (seconds % 3600) / 60) + } else if seconds >= 60 { + format!("{}m{:02}s", seconds / 60, seconds % 60) + } else { + format!("{seconds}s") + }; + match call.verdict.gap_timing { + Some(CacheGapTiming::RecordedEvents) => format!("~{text}"), + _ => text, + } +} + +fn verdict_label(call: &AnalyzedCacheCall) -> String { + let verdict = &call.verdict; + let mut label = match verdict.class { + CacheCallClass::FirstObserved if call.cache_read_tokens == 0 => { + "first observed, cold".to_string() + } + CacheCallClass::FirstObserved => "first observed".to_string(), + CacheCallClass::Boundary => match verdict.boundary { + Some(CacheBoundaryReason::ModelChange) => format!( + "restart: model changed from {}", + verdict.previous_model.as_deref().unwrap_or("unknown") + ), + Some(CacheBoundaryReason::Compaction) => "restart: after compaction".to_string(), + Some(CacheBoundaryReason::EvidenceGap) | None => { + "restart: after unclassifiable usage".to_string() + } + }, + CacheCallClass::BelowThreshold => "too small to judge".to_string(), + CacheCallClass::Comparable => match verdict.loss { + Some(CacheLoss::Full) => "suspected loss, nothing read".to_string(), + Some(CacheLoss::Partial) => "suspected loss, partial reuse".to_string(), + None => "reused".to_string(), + }, + CacheCallClass::Uncached => "cache not used".to_string(), + CacheCallClass::Unclassifiable => format!( + "unclassifiable: {}", + match verdict.unclassifiable { + Some(CacheUnclassifiableReason::TimestampMissing) => "no timestamp", + Some(CacheUnclassifiableReason::Aggregated) => "calls recorded as one total", + Some(CacheUnclassifiableReason::NoInputTelemetry) => "no input counts", + Some(CacheUnclassifiableReason::AmbiguousOrder) => "order not recorded", + None => "unknown", + } + ), + }; + if let Some(ttl) = verdict.previous_write_ttl { + label.push_str(match ttl { + CacheWriteTtl::FiveMinutes => " · previous writes 5m", + CacheWriteTtl::OneHour => " · previous writes 1h", + CacheWriteTtl::Mixed => " · previous writes 5m+1h", + }); + } + label +} + +fn format_share(value: Option) -> String { + value + .map(|value| format!("{:.1}%", value * 100.0)) + .unwrap_or_else(|| "—".to_string()) +} + +fn optional_count(value: Option) -> String { + value.map(format_u64).unwrap_or_else(|| "—".to_string()) +} diff --git a/crates/statsai/src/cli/mod.rs b/crates/statsai/src/cli/mod.rs index cecbb2e..d8012b1 100644 --- a/crates/statsai/src/cli/mod.rs +++ b/crates/statsai/src/cli/mod.rs @@ -2,6 +2,7 @@ pub(super) mod account; pub(super) mod activity; pub(super) mod args; pub(super) mod auth; +pub(super) mod cache; pub(super) mod conversation; pub(super) mod daemon; mod dispatch; diff --git a/crates/statsai/src/cli/report.rs b/crates/statsai/src/cli/report.rs index 1924db0..8559ae1 100644 --- a/crates/statsai/src/cli/report.rs +++ b/crates/statsai/src/cli/report.rs @@ -14,6 +14,9 @@ use super::format::{ }; pub(crate) fn report(command: ReportCommand, store: &Store) -> Result<()> { + if let ReportSubcommand::Cache(args) = command.command { + return super::cache::cache_report(args, store); + } let now = Utc::now(); let (report, json_output, verbose, include_subscriptions) = usage_report_from_command(command, store, now)?; @@ -60,6 +63,7 @@ pub(crate) fn usage_report_from_command( verbose, subscriptions, ), + ReportSubcommand::Cache(_) => bail!("the cache report has its own output"), }; let (since, until) = period.window(now); let (events, summaries) = match period { diff --git a/crates/statsai/src/cli/schema.rs b/crates/statsai/src/cli/schema.rs index b8cdc9a..563b4df 100644 --- a/crates/statsai/src/cli/schema.rs +++ b/crates/statsai/src/cli/schema.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use statsai_core::{QuotaWindowSyncProjectionV1, SyncBatch}; +use statsai_core::{CacheReport, QuotaWindowSyncProjectionV1, SyncBatch}; use super::args::{SchemaCommand, SchemaSubcommand}; @@ -13,6 +13,10 @@ pub(crate) fn schema(command: SchemaCommand) -> Result<()> { let schema = schemars::schema_for!(QuotaWindowSyncProjectionV1); println!("{}", serde_json::to_string_pretty(&schema)?); } + SchemaSubcommand::CacheReport => { + let schema = schemars::schema_for!(CacheReport); + println!("{}", serde_json::to_string_pretty(&schema)?); + } } Ok(()) } diff --git a/crates/statsai/src/cli/sync/batch.rs b/crates/statsai/src/cli/sync/batch.rs index 69367d2..42e75e5 100644 --- a/crates/statsai/src/cli/sync/batch.rs +++ b/crates/statsai/src/cli/sync/batch.rs @@ -14,7 +14,7 @@ use statsai_core::{ SyncAuthoritativeSnapshot, SyncBatch, TaskVerificationCursor, UsageEvent, UsageSummary, SYNC_BATCH_SCHEMA_VERSION, }; -use statsai_store::{QuotaQuery, Store}; +use statsai_store::{summary_for_sync_target, QuotaQuery, Store}; #[cfg(test)] use std::collections::BTreeMap; @@ -46,6 +46,13 @@ pub(crate) fn build_sync_batch_with_identity_key( let batch_id = format!("batch_{}", created_at.timestamp_millis()); let sync_preferences = effective_sync_preferences(store, command)?; let include_projects = sync_preferences.include_projects; + let accepts_cache_health = store.sync_target_accepts_cache_health(&command.sink, target)?; + let for_target = |summary: UsageSummary| { + summary_for_sync_target( + sanitize_summary_for_sync_with_projects(summary, include_projects), + accepts_cache_health, + ) + }; let payload_mode = sync_payload_mode(command)?; let state = if command.sink == "http" || command.since_last { store.sync_state(&command.sink, target)? @@ -81,7 +88,7 @@ pub(crate) fn build_sync_batch_with_identity_key( store .summaries()? .into_iter() - .map(|summary| sanitize_summary_for_sync_with_projects(summary, include_projects)) + .map(for_target) .filter(is_http_rollup_passthrough_summary) .collect() } else { @@ -93,7 +100,7 @@ pub(crate) fn build_sync_batch_with_identity_key( store .summaries_after(summary_cursor)? .into_iter() - .map(|summary| sanitize_summary_for_sync_with_projects(summary, include_projects)) + .map(for_target) .collect() }; let all_sources: Vec<_> = store @@ -323,7 +330,7 @@ pub(crate) fn build_sync_batch_with_identity_key( let all_rollups: Vec<_> = store .all_sync_rollup_summaries()? .into_iter() - .map(|summary| sanitize_summary_for_sync_with_projects(summary, include_projects)) + .map(for_target) .collect(); let current_authoritative_snapshot = SyncAuthoritativeSnapshot { snapshot_id: format!("{batch_id}_authoritative"), @@ -412,11 +419,7 @@ pub(crate) fn build_sync_batch_with_identity_key( } ); summaries.extend(passthrough_summaries); - summaries.extend( - rollups - .into_iter() - .map(|summary| sanitize_summary_for_sync_with_projects(summary, include_projects)), - ); + summaries.extend(rollups.into_iter().map(for_target)); } Ok(( @@ -624,6 +627,9 @@ pub(crate) fn sanitize_source_account_assignment_for_sync( } pub(crate) fn sanitize_event_for_sync(mut event: UsageEvent) -> UsageEvent { + // Per-call timing and sub-agent identity stay on the device. Daily + // summaries carry the counts derived from them. + event.context = None; event.source.source_record_id = None; if let Some(evidence) = event.parse_evidence.as_mut() { evidence.source_line_number = None; diff --git a/crates/statsai/src/cli/sync/http.rs b/crates/statsai/src/cli/sync/http.rs index a8dbccb..12d1675 100644 --- a/crates/statsai/src/cli/sync/http.rs +++ b/crates/statsai/src/cli/sync/http.rs @@ -634,6 +634,15 @@ pub(crate) fn remote_hosted_tasks_enabled(remote: &Value) -> bool { .unwrap_or(true) } +/// Whether the receiver stores `metrics.cache_health` in the version this +/// build writes. Receivers that predate it do not advertise it. +pub(crate) fn remote_accepts_cache_health(remote: Option<&Value>) -> bool { + remote + .and_then(|remote| remote.pointer("/capabilities/cacheHealth")) + .and_then(Value::as_u64) + == Some(u64::from(statsai_core::CACHE_HEALTH_VERSION)) +} + pub(crate) fn remote_code_change_identity_key(remote: &Value) -> Result> { let Some(value) = remote.pointer("/capabilities/codeChangeIdentityKey") else { return Ok(None); diff --git a/crates/statsai/src/cli/sync/mod.rs b/crates/statsai/src/cli/sync/mod.rs index d65e291..cc744cb 100644 --- a/crates/statsai/src/cli/sync/mod.rs +++ b/crates/statsai/src/cli/sync/mod.rs @@ -166,6 +166,11 @@ pub(crate) fn sync(command: SyncCommand, store: &Store, device_id: &str) -> Resu &target, preflight.remote.as_ref(), )?; + store.record_sync_target_cache_health_support( + &command.sink, + &target, + remote_accepts_cache_health(preflight.remote.as_ref()), + )?; Some(preflight) } else { None @@ -633,16 +638,23 @@ pub(crate) fn sync_local_verify( .map(|subscription| subscription.subscription_id.0.as_str()) .collect(), )?; + let accepts_cache_health = store.sync_target_accepts_cache_health(sink, target)?; + let for_target = |summary| { + statsai_store::summary_for_sync_target( + sanitize_summary_for_sync_with_projects(summary, include_projects), + accepts_cache_health, + ) + }; let passthrough_summaries: Vec<_> = store .summaries()? .into_iter() - .map(|summary| sanitize_summary_for_sync_with_projects(summary, include_projects)) + .map(for_target) .filter(is_http_rollup_passthrough_summary) .collect(); let rollup_summaries: Vec<_> = store .all_sync_rollup_summaries()? .into_iter() - .map(|summary| sanitize_summary_for_sync_with_projects(summary, include_projects)) + .map(for_target) .collect(); Ok(SyncLocalVerify { diff --git a/crates/statsai/src/cli/tests/args.rs b/crates/statsai/src/cli/tests/args.rs index 2a984ad..3d862ea 100644 --- a/crates/statsai/src/cli/tests/args.rs +++ b/crates/statsai/src/cli/tests/args.rs @@ -116,3 +116,27 @@ fn price_derived_commands_reprice_and_diagnostic_commands_do_not() { let doctor = Cli::try_parse_from(["statsai", "doctor"]).expect("parse doctor"); assert!(!command_reprices_persisted_usage(&doctor.command)); } + +#[test] +fn cache_report_all_covers_every_stored_call_and_excludes_a_range() { + let cli = Cli::try_parse_from(["statsai", "report", "cache", "--all", "--provider", "codex"]) + .expect("parse --all"); + let Command::Report(ReportCommand { + command: ReportSubcommand::Cache(args), + .. + }) = cli.command + else { + panic!("expected the cache report"); + }; + let now = Utc.with_ymd_and_hms(2026, 7, 10, 12, 0, 0).unwrap(); + let query = cache::cache_report_query(&args, now).expect("query"); + assert_eq!((query.since, query.until), (None, now)); + + for range in ["--from", "--to"] { + assert!( + Cli::try_parse_from(["statsai", "report", "cache", "--all", range, "2026-07-01"]) + .is_err(), + "--all with {range}" + ); + } +} diff --git a/crates/statsai/src/cli/tests/support.rs b/crates/statsai/src/cli/tests/support.rs index b119cdd..d148b9e 100644 --- a/crates/statsai/src/cli/tests/support.rs +++ b/crates/statsai/src/cli/tests/support.rs @@ -264,6 +264,7 @@ pub(super) fn test_event( }, created_at: started_at, imported_at: started_at, + context: None, } } diff --git a/crates/statsai/src/cli/tests/sync/batch.rs b/crates/statsai/src/cli/tests/sync/batch.rs index efe7405..0470ef5 100644 --- a/crates/statsai/src/cli/tests/sync/batch.rs +++ b/crates/statsai/src/cli/tests/sync/batch.rs @@ -2200,3 +2200,65 @@ fn excluding_projects_retires_hosted_sessions_and_excluding_sessions_keeps_them( .expect("sessions on"); assert_eq!(snapshot_session_ids(&command(false, true)), None); } + +#[test] +fn cache_health_reaches_only_receivers_that_advertise_it() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/codex-cache-health"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let mut event = test_event("codex", &source, Utc::now(), None, TokenParts::total(15)); + event.usage.requests = Some(1); + event.context = Some(statsai_core::CallContext { + requested_at: Some(event.created_at), + ..statsai_core::CallContext::default() + }); + store.insert_event(&event).expect("event"); + + let command = SyncCommand { + endpoint: Some("https://api.example.com/api/sync/batches".to_string()), + ..test_sync_command("http") + }; + let target = sync_target(&command).expect("target"); + let has_cache_health = |batch: &SyncBatch| { + batch.summaries.iter().any(|summary| { + summary + .metrics + .as_ref() + .is_some_and(|metrics| metrics.cache_health.is_some()) + }) + }; + + let (batch, _) = build_sync_batch(&command, &store, "device", &target).expect("batch"); + assert!(!has_cache_health(&batch), "unknown receivers get none"); + + store + .record_sync_target_cache_health_support("http", &target, true) + .expect("support"); + let (batch, _) = build_sync_batch(&command, &store, "device", &target).expect("batch"); + assert!(has_cache_health(&batch)); + + let (batch, _) = + build_sync_batch(&test_sync_command("stdout"), &store, "device", "stdout").expect("batch"); + assert!(batch.events.iter().all(|event| event.context.is_none())); +} + +#[test] +fn receivers_advertise_the_cache_health_version_they_store() { + let remote = |value: serde_json::Value| serde_json::json!({ "capabilities": value }); + assert!(remote_accepts_cache_health(Some(&remote( + serde_json::json!({ "cacheHealth": statsai_core::CACHE_HEALTH_VERSION }) + )))); + assert!(!remote_accepts_cache_health(Some(&remote( + serde_json::json!({ "cacheHealth": statsai_core::CACHE_HEALTH_VERSION + 1 }) + )))); + assert!(!remote_accepts_cache_health(Some(&remote( + serde_json::json!({ "hostedTasks": true }) + )))); + assert!(!remote_accepts_cache_health(None)); +} diff --git a/crates/statsai/src/cli/tests/sync/http.rs b/crates/statsai/src/cli/tests/sync/http.rs index 9a4dc6a..4adbbd6 100644 --- a/crates/statsai/src/cli/tests/sync/http.rs +++ b/crates/statsai/src/cli/tests/sync/http.rs @@ -332,14 +332,8 @@ fn http_rollup_sync_restarts_full_snapshot_after_snapshot_failure() { .expect("pending sources") .is_empty()); - let sync_rollups: Vec<_> = store - .all_sync_rollup_summaries() - .expect("rollups") - .into_iter() - .map(sanitize_summary_for_sync) - .collect(); let pending_rollups = store - .pending_summaries_for_sync("http", &target, &sync_rollups) + .pending_http_sync_rollup_summaries(&target) .expect("pending rollups"); assert!(pending_rollups.is_empty()); let state = store diff --git a/crates/statsai/src/cli/tests/sync/mod.rs b/crates/statsai/src/cli/tests/sync/mod.rs index 9ac0fcd..5905dbc 100644 --- a/crates/statsai/src/cli/tests/sync/mod.rs +++ b/crates/statsai/src/cli/tests/sync/mod.rs @@ -643,6 +643,9 @@ fn sync_local_verify_uses_sanitized_rollup_hashes() { store.rebuild_sync_rollups().expect("rebuild"); let target = "https://api.example.com/api/sync/batches".to_string(); + store + .record_sync_target_cache_health_support("http", &target, true) + .expect("receiver support"); let rollups: Vec<_> = store .all_sync_rollup_summaries() .expect("rollups") @@ -700,6 +703,9 @@ fn sync_local_verify_respects_project_sync_opt_in() { store.rebuild_sync_rollups().expect("rebuild"); let target = "https://api.example.com/api/sync/batches".to_string(); + store + .record_sync_target_cache_health_support("http", &target, true) + .expect("receiver support"); let rollups: Vec<_> = store .all_sync_rollup_summaries() .expect("rollups") diff --git a/crates/statsai/src/lib.rs b/crates/statsai/src/lib.rs index bd7852d..06157bb 100644 --- a/crates/statsai/src/lib.rs +++ b/crates/statsai/src/lib.rs @@ -372,6 +372,7 @@ mod tests { }, created_at: started_at, imported_at: started_at, + context: None, }; store.insert_event(&event).expect("legacy event"); assert_eq!(store.applied_pricing_ruleset_version().expect("meta"), None); diff --git a/crates/statsai/tests/cli_surface.rs b/crates/statsai/tests/cli_surface.rs index 2c0bea3..e858006 100644 --- a/crates/statsai/tests/cli_surface.rs +++ b/crates/statsai/tests/cli_surface.rs @@ -114,3 +114,10 @@ fn schema_quota_window_projection_matches_golden() { let golden = include_str!("cli_surface/schema-quota-window-projection.json"); assert_eq!(captured, golden); } + +#[test] +fn schema_cache_report_matches_golden() { + let captured = run_statsai(&["schema", "cache-report"]); + let golden = include_str!("cli_surface/schema-cache-report.json"); + assert_eq!(captured, golden); +} diff --git a/crates/statsai/tests/cli_surface/help.txt b/crates/statsai/tests/cli_surface/help.txt index 7d72338..9183926 100644 --- a/crates/statsai/tests/cli_surface/help.txt +++ b/crates/statsai/tests/cli_surface/help.txt @@ -59,6 +59,7 @@ Commands: monthly Show usage for the last 30 days all-time Show all stored usage range Show usage for an explicit date range + cache Show prompt-cache reuse, suspected cache losses, and context per call help Print this message or the help of the given subcommand(s) Options: @@ -115,6 +116,24 @@ Options: --verbose Show source paths and reasoning tokens --subscriptions Include subscription-value rows -h, --help Print help +===== statsai report cache --help ===== +Show prompt-cache reuse, suspected cache losses, and context per call + +Usage: statsai report cache [OPTIONS] + +Options: + --from Range start (YYYY-MM-DD or RFC3339). Date-only values are UTC days. Defaults to the last 7 days + --store Path to SQLite store + --device-id Device identifier for multi-device sync + --to Range end (YYYY-MM-DD or RFC3339). Defaults to now + --all Report all stored history + --provider Provider filter + --account Provider account id, or `unassigned` + --session Session id (session_…) or a prefix of one + --timeline Add local 10-minute buckets of calls + --details List every call with its verdict and evidence + --json Output as JSON + -h, --help Print help ===== statsai sessions --help ===== List local session rollups @@ -1119,6 +1138,7 @@ Usage: statsai schema [OPTIONS] Commands: sync-batch Print the sync_batch.v6 JSON Schema quota-window-projection Print the quota_window_sync_projection.v1 JSON Schema + cache-report Print the cache_report.v1 JSON Schema help Print this message or the help of the given subcommand(s) Options: @@ -1139,6 +1159,15 @@ Print the quota_window_sync_projection.v1 JSON Schema Usage: statsai schema quota-window-projection [OPTIONS] +Options: + --store Path to SQLite store + --device-id Device identifier for multi-device sync + -h, --help Print help +===== statsai schema cache-report --help ===== +Print the cache_report.v1 JSON Schema + +Usage: statsai schema cache-report [OPTIONS] + Options: --store Path to SQLite store --device-id Device identifier for multi-device sync diff --git a/crates/statsai/tests/cli_surface/schema-cache-report.json b/crates/statsai/tests/cli_surface/schema-cache-report.json new file mode 100644 index 0000000..0bf1f91 --- /dev/null +++ b/crates/statsai/tests/cli_surface/schema-cache-report.json @@ -0,0 +1,863 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CacheReport", + "description": "The prompt-cache report for a period, as `statsai report cache --json` and the local API print it.", + "type": "object", + "required": [ + "boundaries", + "days", + "detector_version", + "diagnostics", + "filters", + "gap_histogram", + "label", + "providers", + "schema_version", + "totals", + "until" + ], + "properties": { + "boundaries": { + "$ref": "#/definitions/CacheBoundaryCounts" + }, + "days": { + "description": "UTC days, matching the daily summaries that sync.", + "type": "array", + "items": { + "$ref": "#/definitions/CacheDayRow" + } + }, + "details": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/AnalyzedCacheCall" + } + }, + "detector_version": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "diagnostics": { + "description": "Detector counts across the providers it covers.", + "allOf": [ + { + "$ref": "#/definitions/CacheHealthV1" + } + ] + }, + "filters": { + "$ref": "#/definitions/CacheReportFilters" + }, + "gap_histogram": { + "type": "array", + "items": { + "$ref": "#/definitions/CacheGapRow" + } + }, + "label": { + "type": "string" + }, + "providers": { + "type": "array", + "items": { + "$ref": "#/definitions/CacheProviderRow" + } + }, + "schema_version": { + "type": "string" + }, + "since": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "timeline": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/CacheTimelineRow" + } + }, + "totals": { + "description": "Token totals across every provider in scope.", + "allOf": [ + { + "$ref": "#/definitions/CacheTokenTotals" + } + ] + }, + "until": { + "type": "string", + "format": "date-time" + } + }, + "definitions": { + "AnalyzedCacheCall": { + "description": "One model call and its verdict, or one unclassifiable block of usage.", + "type": "object", + "required": [ + "cache_read_tokens", + "call_index", + "completed_at", + "event_id", + "input_tokens", + "provider", + "requests", + "session_id", + "source_id", + "verdict" + ], + "properties": { + "agent_hash": { + "type": [ + "string", + "null" + ] + }, + "cache_creation_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "cache_read_tokens": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "call_index": { + "description": "Position inside an aggregated event; zero for single-call events.", + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "completed_at": { + "type": "string", + "format": "date-time" + }, + "event_id": { + "type": "string" + }, + "input_tokens": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "model": { + "type": [ + "string", + "null" + ] + }, + "provider": { + "type": "string" + }, + "provider_account_id": { + "type": [ + "string", + "null" + ] + }, + "requested_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "requests": { + "description": "Model calls this entry stands for: one, or an aggregated event's count.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "session_id": { + "type": "string" + }, + "source_id": { + "type": "string" + }, + "verdict": { + "$ref": "#/definitions/CacheCallVerdict" + }, + "write_ttl": { + "anyOf": [ + { + "$ref": "#/definitions/CacheWriteTtl" + }, + { + "type": "null" + } + ] + } + } + }, + "CacheBoundaryCounts": { + "description": "Boundary counts by reason. Local only: the synced object carries the total.", + "type": "object", + "required": [ + "compaction", + "evidence_gap", + "model_change" + ], + "properties": { + "compaction": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "evidence_gap": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "model_change": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + } + }, + "CacheBoundaryReason": { + "oneOf": [ + { + "description": "The model changed. Prompt caches are per model.", + "type": "string", + "enum": [ + "model_change" + ] + }, + { + "description": "The provider compacted the conversation, which rewrites the prompt.", + "type": "string", + "enum": [ + "compaction" + ] + }, + { + "description": "The previous usage in this stream could not be classified.", + "type": "string", + "enum": [ + "evidence_gap" + ] + } + ] + }, + "CacheCallClass": { + "description": "How the detector classified one model call.", + "oneOf": [ + { + "description": "The first call observed in its stream. Nothing earlier to compare with.", + "type": "string", + "enum": [ + "first_observed" + ] + }, + { + "description": "The comparison restarts here; see [`CacheBoundaryReason`].", + "type": "string", + "enum": [ + "boundary" + ] + }, + { + "description": "The previous call left too little reusable context to judge a loss.", + "type": "string", + "enum": [ + "below_threshold" + ] + }, + { + "description": "Compared with the previous call in its stream.", + "type": "string", + "enum": [ + "comparable" + ] + }, + { + "description": "The call neither read nor wrote the cache, so it did not use it, as when a custom endpoint serves the request. Not judged, and it leaves nothing reusable for the next call.", + "type": "string", + "enum": [ + "uncached" + ] + }, + { + "description": "Usage that cannot be placed in order or split into calls.", + "type": "string", + "enum": [ + "unclassifiable" + ] + } + ] + }, + "CacheCallVerdict": { + "type": "object", + "required": [ + "class" + ], + "properties": { + "baseline_tokens": { + "description": "Reusable context the previous call left, bounded by this call's input.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "boundary": { + "anyOf": [ + { + "$ref": "#/definitions/CacheBoundaryReason" + }, + { + "type": "null" + } + ] + }, + "class": { + "$ref": "#/definitions/CacheCallClass" + }, + "gap_seconds": { + "description": "Seconds since the previous call in the stream completed.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "gap_timing": { + "anyOf": [ + { + "$ref": "#/definitions/CacheGapTiming" + }, + { + "type": "null" + } + ] + }, + "loss": { + "anyOf": [ + { + "$ref": "#/definitions/CacheLoss" + }, + { + "type": "null" + } + ] + }, + "missed_cost_micro_usd": { + "description": "What the missed reuse cost at API prices: the call's cost minus its cost had the missed tokens been cache reads. Set by the caller that prices calls; `None` when the model's pricing is unknown.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "missed_tokens": { + "description": "For a suspected loss, the reusable context it did not read from the cache: the baseline minus its cached reads.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "previous_model": { + "description": "The model before a model change.", + "type": [ + "string", + "null" + ] + }, + "previous_write_ttl": { + "description": "Cache lifetime the previous call's writes asked for, when recorded.", + "anyOf": [ + { + "$ref": "#/definitions/CacheWriteTtl" + }, + { + "type": "null" + } + ] + }, + "unclassifiable": { + "anyOf": [ + { + "$ref": "#/definitions/CacheUnclassifiableReason" + }, + { + "type": "null" + } + ] + } + } + }, + "CacheDayRow": { + "type": "object", + "required": [ + "day", + "totals" + ], + "properties": { + "day": { + "type": "string" + }, + "diagnostics": { + "anyOf": [ + { + "$ref": "#/definitions/CacheHealthV1" + }, + { + "type": "null" + } + ] + }, + "totals": { + "$ref": "#/definitions/CacheTokenTotals" + } + } + }, + "CacheGapRow": { + "type": "object", + "required": [ + "comparable", + "label", + "losses", + "min_minutes", + "missed_tokens" + ], + "properties": { + "comparable": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "label": { + "type": "string" + }, + "losses": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "max_minutes": { + "description": "Exclusive upper bound; `None` for the open last bin.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "min_minutes": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "missed_tokens": { + "description": "Reusable context the bin's suspected losses did not read from the cache.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + } + }, + "CacheGapTiming": { + "description": "What a request gap was measured between.", + "oneOf": [ + { + "description": "From the previous response to the record that sent this request.", + "type": "string", + "enum": [ + "request_start" + ] + }, + { + "description": "Between the two recorded responses. Includes this call's own run time, so it overstates idle time.", + "type": "string", + "enum": [ + "recorded_events" + ] + } + ] + }, + "CacheHealthV1": { + "description": "Bounded prompt-cache diagnostics for one daily summary.\n\nOnly counts leave the device: no timestamps, ids, or per-call values. Token totals stay on the summary's `usage`, so this adds only what they cannot say. `requests` equals `unclassifiable + first_observed + boundaries + below_threshold + uncached + comparable`, and the gap bins add up to `comparable`, `losses`, and `missed_tokens`. Monthly rollups embed every day's object, so the fields most days leave at zero are left out then.", + "type": "object", + "required": [ + "below_threshold", + "boundaries", + "comparable", + "first_cold", + "first_observed", + "full_losses", + "gap_comparable", + "gap_losses", + "losses", + "requests", + "timing_estimated", + "unclassifiable", + "version", + "write_split_unavailable_input_tokens" + ], + "properties": { + "below_threshold": { + "description": "Calls whose predecessor left too little reusable context to judge.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "boundaries": { + "description": "Calls after a model change, compaction, or unclassifiable usage.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "comparable": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "first_cold": { + "description": "First calls that read nothing from the cache.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "first_observed": { + "description": "First calls of their stream.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "full_losses": { + "description": "Suspected losses that read nothing from the cache.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "gap_comparable": { + "description": "Comparable calls by gap since the previous call, one count per bin of [`super::CACHE_GAP_BIN_UPPER_MINUTES`]: `<1m` first, `60m+` last.", + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + }, + "gap_losses": { + "description": "Suspected losses in the same bins.", + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + }, + "gap_missed_tokens": { + "description": "Missed tokens in the same bins.", + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + }, + "losses": { + "description": "Suspected cache losses among comparable calls.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "missed_cost_micro_usd": { + "description": "What that missed reuse cost at API prices, in micro-USD.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "missed_tokens": { + "description": "Reusable context suspected losses did not read from the cache.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "requests": { + "description": "Model calls of providers the detector covers.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "timing_estimated": { + "description": "Comparable calls whose gap was measured between recorded responses rather than from a recorded request start.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "uncached": { + "description": "Calls that neither read nor wrote the cache, so did not use it.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "unclassifiable": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "unpriced_losses": { + "description": "Suspected losses on models without known pricing, left out of `missed_cost_micro_usd`.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "version": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "write_split_unavailable_input_tokens": { + "description": "Ordinary input from calls whose provider did not report cache writes, so it may include writes.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + } + }, + "CacheLoss": { + "oneOf": [ + { + "description": "No cached reads remained.", + "type": "string", + "enum": [ + "full" + ] + }, + { + "description": "Some cached context was still read.", + "type": "string", + "enum": [ + "partial" + ] + } + ] + }, + "CacheProviderRow": { + "type": "object", + "required": [ + "provider", + "totals" + ], + "properties": { + "diagnostics": { + "description": "`None` for providers the detector does not cover.", + "anyOf": [ + { + "$ref": "#/definitions/CacheHealthV1" + }, + { + "type": "null" + } + ] + }, + "provider": { + "type": "string" + }, + "totals": { + "$ref": "#/definitions/CacheTokenTotals" + } + } + }, + "CacheReportFilters": { + "type": "object", + "properties": { + "account": { + "type": [ + "string", + "null" + ] + }, + "provider": { + "type": [ + "string", + "null" + ] + }, + "session": { + "type": [ + "string", + "null" + ] + } + } + }, + "CacheTimelineRow": { + "description": "One local-time bucket of model calls, keyed by when each call completed.", + "type": "object", + "required": [ + "boundaries", + "cache_read_tokens", + "first_observed", + "logical_input_tokens", + "losses", + "missed_tokens", + "requests", + "start" + ], + "properties": { + "boundaries": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "cache_hit_ratio": { + "type": [ + "number", + "null" + ], + "format": "double" + }, + "cache_read_tokens": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "first_observed": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "logical_input_tokens": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "losses": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "missed_tokens": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "requests": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "start": { + "description": "Bucket start in local time, RFC 3339 with offset.", + "type": "string" + } + } + }, + "CacheTokenTotals": { + "type": "object", + "required": [ + "cache_creation_tokens", + "cache_read_tokens", + "input_tokens", + "logical_input_tokens", + "requests" + ], + "properties": { + "avg_logical_input_per_request": { + "type": [ + "number", + "null" + ], + "format": "double" + }, + "cache_creation_tokens": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "cache_hit_ratio": { + "description": "Cache reads over logical input.", + "type": [ + "number", + "null" + ], + "format": "double" + }, + "cache_read_tokens": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "input_tokens": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "logical_input_tokens": { + "description": "Ordinary input, cache writes, and cache reads: the context processed.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "requests": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + } + }, + "CacheUnclassifiableReason": { + "oneOf": [ + { + "description": "The record had no timestamp, so its place in the stream is unknown.", + "type": "string", + "enum": [ + "timestamp_missing" + ] + }, + { + "description": "Several calls were recorded as one total with no per-call breakdown.", + "type": "string", + "enum": [ + "aggregated" + ] + }, + { + "description": "The provider reported no input or cache counts.", + "type": "string", + "enum": [ + "no_input_telemetry" + ] + }, + { + "description": "Another call in the stream has the same timestamp and no recorded order.", + "type": "string", + "enum": [ + "ambiguous_order" + ] + } + ] + }, + "CacheWriteTtl": { + "description": "The cache lifetime a call's writes asked for, as Claude Code records it.", + "type": "string", + "enum": [ + "five_minutes", + "one_hour", + "mixed" + ] + } + } +} diff --git a/crates/statsai/tests/cli_surface/schema-sync-batch.json b/crates/statsai/tests/cli_surface/schema-sync-batch.json index 2610831..f809ac4 100644 --- a/crates/statsai/tests/cli_surface/schema-sync-batch.json +++ b/crates/statsai/tests/cli_surface/schema-sync-batch.json @@ -553,6 +553,148 @@ "custom" ] }, + "CacheHealthV1": { + "description": "Bounded prompt-cache diagnostics for one daily summary.\n\nOnly counts leave the device: no timestamps, ids, or per-call values. Token totals stay on the summary's `usage`, so this adds only what they cannot say. `requests` equals `unclassifiable + first_observed + boundaries + below_threshold + uncached + comparable`, and the gap bins add up to `comparable`, `losses`, and `missed_tokens`. Monthly rollups embed every day's object, so the fields most days leave at zero are left out then.", + "type": "object", + "required": [ + "below_threshold", + "boundaries", + "comparable", + "first_cold", + "first_observed", + "full_losses", + "gap_comparable", + "gap_losses", + "losses", + "requests", + "timing_estimated", + "unclassifiable", + "version", + "write_split_unavailable_input_tokens" + ], + "properties": { + "below_threshold": { + "description": "Calls whose predecessor left too little reusable context to judge.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "boundaries": { + "description": "Calls after a model change, compaction, or unclassifiable usage.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "comparable": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "first_cold": { + "description": "First calls that read nothing from the cache.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "first_observed": { + "description": "First calls of their stream.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "full_losses": { + "description": "Suspected losses that read nothing from the cache.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "gap_comparable": { + "description": "Comparable calls by gap since the previous call, one count per bin of [`super::CACHE_GAP_BIN_UPPER_MINUTES`]: `<1m` first, `60m+` last.", + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + }, + "gap_losses": { + "description": "Suspected losses in the same bins.", + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + }, + "gap_missed_tokens": { + "description": "Missed tokens in the same bins.", + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + }, + "losses": { + "description": "Suspected cache losses among comparable calls.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "missed_cost_micro_usd": { + "description": "What that missed reuse cost at API prices, in micro-USD.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "missed_tokens": { + "description": "Reusable context suspected losses did not read from the cache.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "requests": { + "description": "Model calls of providers the detector covers.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "timing_estimated": { + "description": "Comparable calls whose gap was measured between recorded responses rather than from a recorded request start.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "uncached": { + "description": "Calls that neither read nor wrote the cache, so did not use it.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "unclassifiable": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "unpriced_losses": { + "description": "Suspected losses on models without known pricing, left out of `missed_cost_micro_usd`.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "version": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "write_split_unavailable_input_tokens": { + "description": "Ordinary input from calls whose provider did not report cache writes, so it may include writes.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + } + }, "CodeChangeMetric": { "type": "object", "required": [ @@ -2151,6 +2293,17 @@ "format": "uint64", "minimum": 0.0 }, + "cache_health": { + "description": "Prompt-cache diagnostics, for providers whose events record call order.", + "anyOf": [ + { + "$ref": "#/definitions/CacheHealthV1" + }, + { + "type": "null" + } + ] + }, "cache_hit_ratio": { "anyOf": [ { diff --git a/crates/statsai/tests/report_cli.rs b/crates/statsai/tests/report_cli.rs index b139515..4ca7bc6 100644 --- a/crates/statsai/tests/report_cli.rs +++ b/crates/statsai/tests/report_cli.rs @@ -89,6 +89,7 @@ fn test_event( }, created_at: started_at, imported_at: started_at, + context: None, } }