diff --git a/Cargo.lock b/Cargo.lock index 7fc408a..d676a2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2929,6 +2929,7 @@ dependencies = [ "libc", "log", "notify", + "percent-encoding", "serde", "serde_json", "statsai-adapters", diff --git a/README.md b/README.md index 1670ac0..af87a47 100644 --- a/README.md +++ b/README.md @@ -323,6 +323,28 @@ The benchmark becomes a meaningful quality gate after you record verified ground truth. See [`docs/task-collection.md`](docs/task-collection.md) and [`docs/task-benchmarking.md`](docs/task-benchmarking.md). +### See how well the prompt cache is working + +```sh +statsai report cache +statsai report cache --from 2026-06-01 --to 2026-06-07 --provider claude_code +statsai report cache --all --provider codex +statsai report cache --session session_1a2b --details +statsai report cache --timeline --json +``` + +The cache report shows how much logical input (ordinary input, cache writes, +and cache reads) was served from the prompt cache, how much context each model +call processed, and when reuse dropped. For Claude Code and Codex it also walks +each agent's calls in order: a call whose cached reads fall at least half, and +at least 4,096 tokens, below what the previous call left reusable is a +suspected cache loss. Losses are grouped by the gap since the previous call, so +drops after a pause read apart from drops during active use. It is a heuristic +over recorded token counts, not a provider-confirmed miss, and calls that +cannot be ordered or split are reported as unclassifiable rather than as +healthy. `--timeline` adds local 10-minute buckets and `--details` lists every +call with its evidence; per-call details never leave the device. + ### Keep a durable conversation archive Collect complete conversations into the local SQLite store, then search or @@ -394,6 +416,7 @@ statsai report monthly --subscriptions statsai report all-time --json --verbose statsai report range --from 2026-01-01 --to 2026-03-31 statsai report range --from 2026-05-01 --json +statsai report cache --from 2026-05-01 --details ``` Normal scans use a per-source file signature cache. `--no-cache` forces a @@ -439,6 +462,7 @@ statsai sync --sink http --since-last statsai sync --sink http --verify statsai sync --status statsai schema sync-batch +statsai schema cache-report ``` HTTP sync uses the stored device session unless `--auth-token` or @@ -459,6 +483,10 @@ curl -H "Authorization: Bearer $(cat ~/.statsai/daemon-token)" \ http://127.0.0.1:8765/accounts ``` +`/reports/cache` returns the `statsai report cache --json` report and accepts +`from`, `to`, `all`, `provider`, `account`, `session`, and `timeline` query +parameters. + Browser-originated requests are rejected. Sync writes must use `Content-Type: application/json` and stay below 8 MiB. diff --git a/crates/statsai-adapters/src/cache.rs b/crates/statsai-adapters/src/cache.rs index e3b9edd..3666c18 100644 --- a/crates/statsai-adapters/src/cache.rs +++ b/crates/statsai-adapters/src/cache.rs @@ -15,11 +15,15 @@ pub(crate) const SCAN_CACHE_SIGNATURE_VERSION: &str = "scan-cache.v1"; // parent's name, and turns without a reported duration end at their last work. // session-metadata.v36: sub-agents report the parent they borrow a name from, // so the name is joined when the session is built. -pub(crate) const CODEX_SCAN_CACHE_PARSER_REVISION: &str = "session-metadata.v36"; +// call-context.v38: turn events list their model calls, with request starts, +// compaction boundaries, and each call's model, for the prompt-cache report. +pub(crate) const CODEX_SCAN_CACHE_PARSER_REVISION: &str = "call-context.v38"; // session-metadata.v30: Claude user-message counts include only typed prompts, // not tool results, meta lines, or compaction summaries. (v29: events carry the // session's custom or AI title and the prompt that started their turn.) -pub(crate) const CLAUDE_SCAN_CACHE_PARSER_REVISION: &str = "session-metadata.v30"; +// call-context.v31: events record their request start, sub-agent, and +// compaction boundary for the prompt-cache report. +pub(crate) const CLAUDE_SCAN_CACHE_PARSER_REVISION: &str = "call-context.v31"; // session-metadata.v21: events carry each session's user and assistant message // counts. (v20: session rows report their message count as requests and are // priced per message, so long-context tiers stop being decided session-wide.) diff --git a/crates/statsai-adapters/src/claude/parse.rs b/crates/statsai-adapters/src/claude/parse.rs index d27275a..fb87194 100644 --- a/crates/statsai-adapters/src/claude/parse.rs +++ b/crates/statsai-adapters/src/claude/parse.rs @@ -50,6 +50,7 @@ pub(crate) fn parse_claude_file( // counted on the first snapshot have to ride along with that replacement. let mut last_usage_users = HashMap::::new(); let mut turn_started_at = HashMap::>::new(); + let mut call_streams = HashMap::<(String, Option), ClaudeCallStream>::new(); loop { let line_status = read_bounded_jsonl_line(&mut reader, &mut line_bytes, MAX_JSONL_RECORD_BYTES)?; @@ -84,6 +85,11 @@ pub(crate) fn parse_claude_file( current_reasoning = reasoning; } let session_raw = claude_session_raw(&value, path); + let agent_hash = claude_agent_hash(&value); + call_streams + .entry((session_raw.clone(), agent_hash.clone())) + .or_default() + .observe(&value); if let Some((rank, title)) = claude_session_title_from_value(&value) { session_titles.remember(session_raw, rank, title); continue; @@ -176,7 +182,7 @@ pub(crate) fn parse_claude_file( model, usage, runtime: message_count_runtime(user_messages, 1, 0), - session_raw, + session_raw: session_raw.clone(), project, event_kind: "claude_message_usage", source_file: path, @@ -189,6 +195,9 @@ pub(crate) fn parse_claude_file( }, ); event.session.turn_started_at = prompted_at.filter(|prompted_at| *prompted_at <= timestamp); + event.context = call_streams + .get_mut(&(session_raw, agent_hash.clone())) + .and_then(|stream| stream.call_context(record_id.as_deref(), timestamp, agent_hash)); push_deduped(ctx.scan, ctx.seen, event, duplicate_selection); } @@ -204,6 +213,121 @@ pub(crate) fn parse_claude_file( Ok(()) } +/// Prompt-cache context for one agent's calls in one session. +/// +/// A request goes out when Claude Code records the prompt or tool result that +/// triggers it, so the latest such record before a response is the request +/// start. A streamed response is written as several records sharing one +/// provider-record id; all of them describe the same call, and so does any +/// later copy of it, which keeps the marks its first record got. +/// +/// Transcripts are not in time order: compaction rewrites the messages it +/// keeps after its boundary, with their original timestamps. Everything here +/// therefore follows file order, and a record older than the compaction +/// boundary is one of those copies, not the call that follows it. +#[derive(Default)] +struct ClaudeCallStream { + last_input_at: Option>, + last_completed_at: Option>, + pending_compaction: Option, + marks_by_record: HashMap, +} + +#[derive(Clone, Copy)] +enum ClaudeCompaction { + At(chrono::DateTime), + Unplaced, +} + +#[derive(Clone, Copy)] +struct ClaudeCallMarks { + requested_at: Option>, + after_compaction: bool, +} + +impl ClaudeCallStream { + fn observe(&mut self, value: &Value) { + if claude_record_is_compact_boundary(value) { + self.pending_compaction = Some( + timestamp_from_nested_value(value) + .map_or(ClaudeCompaction::Unplaced, ClaudeCompaction::At), + ); + } else if claude_record_is_user(value) { + if let Some(timestamp) = timestamp_from_nested_value(value) { + self.last_input_at = Some(timestamp); + } + } + } + + fn call_context( + &mut self, + record_id: Option<&str>, + completed_at: chrono::DateTime, + agent_hash: Option, + ) -> Option { + let known = record_id.and_then(|record_id| self.marks_by_record.get(record_id)); + let marks = match known { + Some(marks) => { + // A later record of a known call: another streamed part, or a + // copy compaction wrote with the original timestamp. + let marks = *marks; + self.last_completed_at = self.last_completed_at.max(Some(completed_at)); + marks + } + None => { + // Input recorded before the previous response belongs to an + // earlier exchange, as when a call retries without new input. + let previous = self.last_completed_at; + let after_compaction = match self.pending_compaction { + Some(ClaudeCompaction::At(at)) if completed_at < at => false, + Some(_) => { + self.pending_compaction = None; + true + } + None => false, + }; + let marks = ClaudeCallMarks { + requested_at: self.last_input_at.filter(|at| { + *at <= completed_at && previous.is_none_or(|previous| *at >= previous) + }), + after_compaction, + }; + if let Some(record_id) = record_id { + self.marks_by_record.insert(record_id.to_string(), marks); + } + self.last_completed_at = Some(completed_at); + marks + } + }; + let context = statsai_core::CallContext { + agent_hash, + requested_at: marks.requested_at, + after_compaction: marks.after_compaction, + calls: Vec::new(), + }; + (context != statsai_core::CallContext::default()).then_some(context) + } +} + +/// Sub-agent identity. Sub-agents write their own transcripts, marked as +/// sidechains, under the parent's session id. +fn claude_agent_hash(value: &Value) -> Option { + if let Some(agent_id) = value + .get("agentId") + .and_then(Value::as_str) + .filter(|agent_id| !agent_id.trim().is_empty()) + { + return Some(hash_text(&format!("claude_agent:{agent_id}"))[..24].to_string()); + } + (value.get("isSidechain").and_then(Value::as_bool) == Some(true)) + .then(|| hash_text("claude_agent:sidechain")[..24].to_string()) +} + +fn claude_record_is_compact_boundary(value: &Value) -> bool { + value.get("type").and_then(Value::as_str) == Some("system") + && value.get("subtype").and_then(Value::as_str) == Some("compact_boundary") +} + /// Session names found across one source's transcripts. /// /// A resumed or forked session copies earlier messages into a new transcript, diff --git a/crates/statsai-adapters/src/claude/tests/parse.rs b/crates/statsai-adapters/src/claude/tests/parse.rs index 9cf184e..5570abf 100644 --- a/crates/statsai-adapters/src/claude/tests/parse.rs +++ b/crates/statsai-adapters/src/claude/tests/parse.rs @@ -1373,3 +1373,204 @@ fn claude_title_only_transcripts_report_session_names() { }] ); } + +#[test] +fn claude_events_record_request_starts_sub_agents_and_compaction() { + let dir = tempfile::tempdir().expect("tempdir"); + let project = dir.path().join("projects").join("example"); + let agents = project.join("session-1").join("subagents"); + std::fs::create_dir_all(&agents).expect("projects"); + let assistant = |timestamp: &str, message: &str, request: &str, extra: &str| { + format!( + r#"{{"timestamp":"{timestamp}","sessionId":"session-1"{extra},"type":"assistant","requestId":"{request}","message":{{"id":"{message}","role":"assistant","model":"claude-opus-5","usage":{{"input_tokens":2,"cache_creation_input_tokens":100,"cache_read_input_tokens":9000,"output_tokens":5}}}}}}"# + ) + }; + let main = [ + r#"{"timestamp":"2026-08-05T10:00:00Z","sessionId":"session-1","type":"user","message":{"role":"user","content":"hello"}}"#.to_string(), + assistant("2026-08-05T10:00:05Z", "m1", "r1", ""), + assistant("2026-08-05T10:00:07Z", "m1", "r1", ""), + r#"{"timestamp":"2026-08-05T10:00:20Z","sessionId":"session-1","type":"user","toolUseResult":{},"message":{"role":"user","content":[{"type":"tool_result","content":"ok"}]}}"#.to_string(), + r#"{"timestamp":"2026-08-05T10:00:30Z","sessionId":"session-1","type":"system","subtype":"compact_boundary"}"#.to_string(), + r#"{"timestamp":"2026-08-05T10:00:31Z","sessionId":"session-1","type":"user","isCompactSummary":true,"message":{"role":"user","content":"summary"}}"#.to_string(), + assistant("2026-08-05T10:00:40Z", "m2", "r2", ""), + assistant("2026-08-05T10:00:50Z", "m4", "r4", ""), + ]; + std::fs::write(project.join("session-1.jsonl"), main.join("\n") + "\n").expect("main"); + let agent = [ + r#"{"timestamp":"2026-08-05T10:00:08Z","sessionId":"session-1","agentId":"a1","isSidechain":true,"type":"user","message":{"role":"user","content":"task"}}"#.to_string(), + assistant( + "2026-08-05T10:00:12Z", + "m3", + "r3", + r#","agentId":"a1","isSidechain":true"#, + ), + ]; + std::fs::write(agents.join("agent-a1.jsonl"), agent.join("\n") + "\n").expect("agent"); + + let source = SourceLocation::local_adapter( + CLAUDE_CODE_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let scan = scan_claude_source(&ClaudeCodeAdapter, &source, &options()).expect("scan"); + let event_at = |timestamp: &str| { + let at = timestamp + .parse::>() + .expect("timestamp"); + scan.events + .iter() + .find(|event| event.created_at == at) + .unwrap_or_else(|| panic!("event at {timestamp}")) + }; + let at = |timestamp: &str| Some(timestamp.parse::>().expect("time")); + + assert_eq!(scan.events.len(), 4, "streamed snapshots are one call"); + let first = event_at("2026-08-05T10:00:07Z") + .context + .as_ref() + .expect("context"); + assert_eq!(first.requested_at, at("2026-08-05T10:00:00Z")); + assert!(!first.after_compaction); + assert_eq!(first.agent_hash, None); + + let compacted = event_at("2026-08-05T10:00:40Z") + .context + .as_ref() + .expect("context"); + assert_eq!(compacted.requested_at, at("2026-08-05T10:00:31Z")); + assert!(compacted.after_compaction); + + // No input since the previous response: no request start, and the + // compaction marked only the call after it. + assert_eq!(event_at("2026-08-05T10:00:50Z").context, None); + + let sub_agent = event_at("2026-08-05T10:00:12Z") + .context + .as_ref() + .expect("context"); + assert!(sub_agent.agent_hash.is_some()); + assert_eq!(sub_agent.requested_at, at("2026-08-05T10:00:08Z")); +} + +#[test] +fn claude_compaction_copies_do_not_shift_request_starts_or_the_boundary() { + let dir = tempfile::tempdir().expect("tempdir"); + let project = dir.path().join("projects").join("example"); + std::fs::create_dir_all(&project).expect("projects"); + let user = |timestamp: &str| { + format!( + r#"{{"timestamp":"{timestamp}","sessionId":"session-1","type":"user","message":{{"role":"user","content":"go"}}}}"# + ) + }; + let assistant = |timestamp: &str, message: &str| { + format!( + r#"{{"timestamp":"{timestamp}","sessionId":"session-1","type":"assistant","requestId":"r-{message}","message":{{"id":"{message}","role":"assistant","model":"claude-opus-5","usage":{{"input_tokens":2,"cache_read_input_tokens":9000,"output_tokens":5}}}}}}"# + ) + }; + let lines = [ + user("2026-08-05T10:00:00Z"), + assistant("2026-08-05T10:00:05Z", "m1"), + user("2026-08-05T11:00:00Z"), + assistant("2026-08-05T11:00:05Z", "m2"), + r#"{"timestamp":"2026-08-05T11:30:00Z","sessionId":"session-1","type":"system","subtype":"compact_boundary"}"#.to_string(), + // The preserved segment, rewritten with its original timestamps. + user("2026-08-05T10:00:00Z"), + assistant("2026-08-05T10:00:05Z", "m1"), + user("2026-08-05T11:31:00Z"), + assistant("2026-08-05T11:31:05Z", "m3"), + ]; + std::fs::write(project.join("session-1.jsonl"), lines.join("\n") + "\n").expect("transcript"); + let source = SourceLocation::local_adapter( + CLAUDE_CODE_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let scan = scan_claude_source(&ClaudeCodeAdapter, &source, &options()).expect("scan"); + let context = |timestamp: &str| { + let at = timestamp + .parse::>() + .expect("timestamp"); + scan.events + .iter() + .find(|event| event.created_at == at) + .and_then(|event| event.context.clone()) + .unwrap_or_default() + }; + + assert_eq!(scan.events.len(), 3); + assert!( + !context("2026-08-05T10:00:05Z").after_compaction, + "a copy is not the next call" + ); + let next = context("2026-08-05T11:31:05Z"); + assert!(next.after_compaction); + assert_eq!( + next.requested_at, + Some("2026-08-05T11:31:00Z".parse().expect("timestamp")) + ); +} + +#[test] +fn claude_compaction_copies_keep_the_marks_of_the_call_they_copy() { + let dir = tempfile::tempdir().expect("tempdir"); + let project = dir.path().join("projects").join("example"); + std::fs::create_dir_all(&project).expect("projects"); + let user = |timestamp: &str| { + format!( + r#"{{"timestamp":"{timestamp}","sessionId":"session-1","type":"user","message":{{"role":"user","content":"go"}}}}"# + ) + }; + let assistant = |timestamp: &str, message: &str| { + format!( + r#"{{"timestamp":"{timestamp}","sessionId":"session-1","type":"assistant","requestId":"r-{message}","message":{{"id":"{message}","role":"assistant","model":"claude-opus-5","usage":{{"input_tokens":2,"cache_read_input_tokens":9000,"output_tokens":5}}}}}}"# + ) + }; + let boundary = |timestamp: &str| { + format!( + r#"{{"timestamp":"{timestamp}","sessionId":"session-1","type":"system","subtype":"compact_boundary"}}"# + ) + }; + let lines = [ + user("2026-08-05T10:00:00Z"), + assistant("2026-08-05T10:00:05Z", "m1"), + boundary("2026-08-05T10:30:00Z"), + user("2026-08-05T10:31:00Z"), + assistant("2026-08-05T10:31:05Z", "m2"), + boundary("2026-08-05T11:30:00Z"), + // A second compaction keeps the call that followed the first one. + user("2026-08-05T10:31:00Z"), + assistant("2026-08-05T10:31:05Z", "m2"), + ]; + std::fs::write(project.join("session-1.jsonl"), lines.join("\n") + "\n").expect("transcript"); + let source = SourceLocation::local_adapter( + CLAUDE_CODE_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let scan = scan_claude_source(&ClaudeCodeAdapter, &source, &options()).expect("scan"); + let at = "2026-08-05T10:31:05Z" + .parse::>() + .expect("timestamp"); + let copied = scan + .events + .iter() + .filter(|event| event.created_at == at) + .collect::>(); + + assert_eq!(copied.len(), 1); + let context = copied[0].context.clone().unwrap_or_default(); + assert!( + context.after_compaction, + "the copy keeps its compaction mark" + ); + assert_eq!( + context.requested_at, + Some("2026-08-05T10:31:00Z".parse().expect("timestamp")) + ); +} diff --git a/crates/statsai-adapters/src/codex/parse/line.rs b/crates/statsai-adapters/src/codex/parse/line.rs index c19a3ff..9e63d59 100644 --- a/crates/statsai-adapters/src/codex/parse/line.rs +++ b/crates/statsai-adapters/src/codex/parse/line.rs @@ -48,6 +48,8 @@ pub(crate) enum CodexLineKind { TokenCount, TokenUsageRecord, Compacted, + /// A remote compaction's replacement history, written as a response item. + ResponseItemCompaction, TaskStarted, TaskComplete, HeadlessUsage, @@ -114,6 +116,8 @@ pub(crate) fn codex_line_kind(line: &str) -> CodexLineKind { || header.contains("\"payload\":{\"type\":\"tool_search_call\"") { CodexLineKind::ResponseItemToolCall + } else if header.contains("\"payload\":{\"type\":\"compaction\"") { + CodexLineKind::ResponseItemCompaction } else { CodexLineKind::Irrelevant }; diff --git a/crates/statsai-adapters/src/codex/parse/mod.rs b/crates/statsai-adapters/src/codex/parse/mod.rs index 054d0fa..1ea485f 100644 --- a/crates/statsai-adapters/src/codex/parse/mod.rs +++ b/crates/statsai-adapters/src/codex/parse/mod.rs @@ -54,6 +54,7 @@ pub(crate) fn parse_codex_file( let mut line_bytes = Vec::new(); let mut index = 0usize; let mut activity = crate::activity::CodexActivityExtractor::default(); + let mut call_clock = CodexCallClock::default(); loop { let line_status = @@ -65,6 +66,18 @@ pub(crate) fn parse_codex_file( if line_status == BoundedLineRead::Oversized { ctx.scan.diagnostics.raw_rows += 1; ctx.scan.diagnostics.oversized_rows += 1; + // A compaction row carries the whole replacement history and can + // outgrow the record limit. Its header still marks the boundary. + let header = String::from_utf8_lossy(&line_bytes); + if matches!( + codex_line_kind(&header), + CodexLineKind::Compacted | CodexLineKind::ResponseItemCompaction + ) { + let session = + codex_json_string_prefix_after_marker(&header, "\"session_id\":\"", 256) + .unwrap_or_else(|| session_raw.clone()); + call_clock.compaction(&session); + } continue; } let Ok(line) = std::str::from_utf8(&line_bytes) else { @@ -95,6 +108,18 @@ pub(crate) fn parse_codex_file( continue; } if line_kind == CodexLineKind::ResponseItemToolCall { + let header = codex_line_header(line); + if header.contains("\"payload\":{\"type\":\"function_call_output\"") + || header.contains("\"payload\":{\"type\":\"custom_tool_call_output\"") + { + if let (timestamp, false) = codex_timestamp_from_text( + codex_json_string_prefix_after_marker(header, "\"timestamp\":\"", 64) + .as_deref(), + fallback_timestamp, + ) { + call_clock.input(&session_raw, timestamp); + } + } let activity_started_at = std::time::Instant::now(); activity.observe_legacy_line( ctx.source, @@ -118,6 +143,13 @@ pub(crate) fn parse_codex_file( ) .unwrap_or_else(|| session_raw.clone()); unpaired_records.remove(&session); + call_clock.compaction(&session); + continue; + } + if line_kind == CodexLineKind::ResponseItemCompaction { + // Pairing is left alone: unlike `compacted`, this row does not end + // a token_usage_record's pairing window. + call_clock.compaction(&session_raw); continue; } if line_kind == CodexLineKind::Irrelevant && !is_codex_quota_line_structurally(line) { @@ -148,6 +180,8 @@ pub(crate) fn parse_codex_file( ); if timestamp_inferred { ctx.scan.diagnostics.timestamp_fallbacks += 1; + } else if codex_role_sends_input(role.as_deref()) { + call_clock.input(&session_raw, timestamp); } let mut model_inferred = false; let model = current_model @@ -197,6 +231,8 @@ pub(crate) fn parse_codex_file( task_completed_at: None, task_duration_ms: None, time_to_first_token_ms: None, + requested_at: None, + after_compaction: false, }); continue; } @@ -208,6 +244,8 @@ pub(crate) fn parse_codex_file( codex_timestamp_from_text(parsed.timestamp.as_deref(), fallback_timestamp); if timestamp_inferred { ctx.scan.diagnostics.timestamp_fallbacks += 1; + } else if codex_role_sends_input(parsed.payload.role.as_deref()) { + call_clock.input(&session_raw, timestamp); } let mut model_inferred = false; let model = current_model @@ -263,18 +301,23 @@ pub(crate) fn parse_codex_file( task_completed_at: None, task_duration_ms: None, time_to_first_token_ms: None, + requested_at: None, + after_compaction: false, }); continue; } if line_kind == CodexLineKind::EventUserMessage { - if !collect_tasks { - continue; - } let header = codex_line_header(line); let (timestamp, timestamp_inferred) = codex_timestamp_from_text( codex_json_string_prefix_after_marker(header, "\"timestamp\":\"", 64).as_deref(), fallback_timestamp, ); + if !timestamp_inferred { + call_clock.input(&session_raw, timestamp); + } + if !collect_tasks { + continue; + } if timestamp_inferred { ctx.scan.diagnostics.timestamp_fallbacks += 1; } @@ -326,6 +369,8 @@ pub(crate) fn parse_codex_file( task_completed_at: None, task_duration_ms: None, time_to_first_token_ms: None, + requested_at: None, + after_compaction: false, }); continue; } @@ -456,6 +501,11 @@ pub(crate) fn parse_codex_file( .unwrap_or((fallback_timestamp, true)); if timestamp_inferred { ctx.scan.diagnostics.timestamp_fallbacks += 1; + } else if is_task_started { + call_clock.input( + &event_session_raw, + task_started_at.unwrap_or(timestamp).min(timestamp), + ); } if let Some(quota) = codex_quota_observation( ctx.source, @@ -514,6 +564,12 @@ pub(crate) fn parse_codex_file( } }); + let (requested_at, after_compaction) = match usage.as_ref() { + Some(_) if !timestamp_inferred => { + call_clock.call(&event_session_raw, timestamp, is_token_usage_record) + } + _ => (None, false), + }; records.push(CodexLineRecord { line_number: index, timestamp, @@ -538,6 +594,8 @@ pub(crate) fn parse_codex_file( task_completed_at, task_duration_ms, time_to_first_token_ms, + requested_at, + after_compaction, }); } @@ -568,6 +626,12 @@ pub(crate) fn parse_codex_file( .unwrap_or_default(), quota_lines: Vec::new(), project: record.project.clone(), + calls: record + .usage + .as_ref() + .map(|usage| codex_model_call(record, usage)) + .into_iter() + .collect(), }); if record.usage.is_some() { consumed_usage_lines.insert(record.line_number); @@ -659,6 +723,7 @@ pub(crate) fn parse_codex_file( .map(|accumulated| sum_usage_counts(accumulated, &usage)) .unwrap_or_else(|| usage.clone()), ); + turn.calls.push(codex_model_call(record, &usage)); turn.last_usage = Some(usage); turn.usage_lines.push(record.line_number); } @@ -745,6 +810,24 @@ pub(crate) fn parse_codex_file( }, ); event.session.title = turn.title.clone(); + // The completion's own usage, when present, replaces the calls', + // but a compaction anywhere in the turn still came before it. + if record.usage.is_none() && !turn.calls.is_empty() { + event.context = Some(CallContext { + calls: turn.calls.clone(), + ..CallContext::default() + }); + } else if record.usage.is_some() { + let after_compaction = + record.after_compaction || turn.calls.iter().any(|call| call.after_compaction); + if record.requested_at.is_some() || after_compaction { + event.context = Some(CallContext { + requested_at: record.requested_at, + after_compaction, + ..CallContext::default() + }); + } + } let mut linked_quota_lines = turn.usage_lines.clone(); linked_quota_lines.extend_from_slice(&turn.quota_lines); if record.usage.is_some() { @@ -979,6 +1062,13 @@ pub(crate) fn parse_codex_file( }, ); event.session.title = session_title; + if record.requested_at.is_some() || record.after_compaction { + event.context = Some(CallContext { + requested_at: record.requested_at, + after_compaction: record.after_compaction, + ..CallContext::default() + }); + } // A record's paired token_count carries the quota sample for it. let mut linked_quota_lines = vec![record.line_number]; linked_quota_lines.extend(paired_quota_lines.get(&record.line_number)); @@ -1003,3 +1093,24 @@ pub(crate) fn parse_codex_file( Ok(()) } + +/// Prompts and developer instructions are input to the next request. +fn codex_role_sends_input(role: Option<&str>) -> bool { + matches!(role, Some("user" | "developer")) +} + +fn codex_model_call(record: &CodexLineRecord, usage: &UsageCounts) -> ModelCall { + ModelCall { + completed_at: record.timestamp, + requested_at: record.requested_at, + input_tokens: usage.input_tokens, + cache_read_tokens: usage.cache_read_tokens, + cache_creation_tokens: usage.cache_creation_tokens, + after_compaction: record.after_compaction, + model: record + .model + .as_ref() + .filter(|_| !record.model_inferred) + .and_then(statsai_core::cache_model_name), + } +} diff --git a/crates/statsai-adapters/src/codex/parse/types.rs b/crates/statsai-adapters/src/codex/parse/types.rs index ba0d1aa..a88368c 100644 --- a/crates/statsai-adapters/src/codex/parse/types.rs +++ b/crates/statsai-adapters/src/codex/parse/types.rs @@ -23,6 +23,10 @@ pub(crate) struct CodexLineRecord { pub(crate) task_completed_at: Option>, pub(crate) task_duration_ms: Option, pub(crate) time_to_first_token_ms: Option, + /// For usage lines: when the request was sent, if the rollout says. + pub(crate) requested_at: Option>, + /// For usage lines: the conversation was compacted before this call. + pub(crate) after_compaction: bool, } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -117,4 +121,50 @@ pub(crate) struct ActiveCodexTurn { /// observations still belong to this turn's event. pub(crate) quota_lines: Vec, pub(crate) project: Option, + /// The model calls whose usage the turn accumulates, in order. + pub(crate) calls: Vec, +} + +/// When each session's model calls were requested, for the prompt-cache +/// report. +/// +/// A `token_usage_record` is written as the response completes, before any +/// tool runs, so the latest prompt, tool output, or turn start after it is +/// when the next request went out. A `token_count` alone is written after the +/// response's tools have run, so it cannot place a request start. Rollouts can +/// carry copied history with older timestamps, so this follows file order. +#[derive(Default)] +pub(crate) struct CodexCallClock { + last_input_at: HashMap>, + last_call_at: HashMap>, + pending_compaction: HashSet, +} + +impl CodexCallClock { + pub(crate) fn input(&mut self, session_raw: &str, at: DateTime) { + self.last_input_at.insert(session_raw.to_string(), at); + } + + pub(crate) fn compaction(&mut self, session_raw: &str) { + self.pending_compaction.insert(session_raw.to_string()); + } + + /// Records a call completing and returns its request start, when the + /// rollout places one, and whether it follows a compaction. + pub(crate) fn call( + &mut self, + session_raw: &str, + completed_at: DateTime, + record_placed: bool, + ) -> (Option>, bool) { + let previous_call_at = self + .last_call_at + .insert(session_raw.to_string(), completed_at); + let requested_at = record_placed + .then(|| self.last_input_at.get(session_raw).copied()) + .flatten() + .filter(|at| *at <= completed_at) + .filter(|at| previous_call_at.is_none_or(|previous| *at >= previous)); + (requested_at, self.pending_compaction.remove(session_raw)) + } } diff --git a/crates/statsai-adapters/src/codex/tests/parse/calls.rs b/crates/statsai-adapters/src/codex/tests/parse/calls.rs new file mode 100644 index 0000000..a24300c --- /dev/null +++ b/crates/statsai-adapters/src/codex/tests/parse/calls.rs @@ -0,0 +1,219 @@ +use super::*; + +/// Codex reports cached input inside `input_tokens`. +fn usage_record(timestamp: &str, input: u64, cached: u64) -> String { + format!( + r#"{{"timestamp":"{timestamp}","type":"token_usage_record","payload":{{"usage":{{"input_tokens":{input},"cached_input_tokens":{cached},"cache_write_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0,"total_tokens":{}}}}}}}"#, + input + 50 + ) +} + +fn token_count(timestamp: &str, total: u64, input: u64, cached: u64) -> String { + format!( + r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"token_count","info":{{"total_token_usage":{{"input_tokens":{total},"cached_input_tokens":0,"output_tokens":0,"total_tokens":{total}}},"last_token_usage":{{"input_tokens":{input},"cached_input_tokens":{cached},"cache_write_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0,"total_tokens":{}}}}}}}}}"#, + input + 50 + ) +} + +fn scan_lines(lines: &[String]) -> AdapterScan { + let dir = tempfile::tempdir().expect("tempdir"); + let sessions = dir.path().join("sessions"); + std::fs::create_dir_all(&sessions).expect("sessions"); + std::fs::write(sessions.join("rollout.jsonl"), lines.join("\n") + "\n").expect("fixture"); + let source = SourceLocation::local_adapter( + CODEX_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + scan_codex_source(&CodexAdapter, &source, &options_without_tasks()).expect("scan") +} + +fn at(timestamp: &str) -> Option> { + Some(timestamp.parse().expect("timestamp")) +} + +fn turn_lines(compaction: String) -> Vec { + vec![ + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"session_meta","payload":{"id":"thread-1"}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"event_msg","payload":{"type":"task_started","turn_id":"turn-1"}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:01Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"go"}]}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:09Z","type":"response_item","payload":{"type":"custom_tool_call","name":"apply_patch","input":""}}"#.to_string(), + usage_record("2026-09-01T10:00:10Z", 30_000, 0), + r#"{"timestamp":"2026-09-01T10:00:12Z","type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"c1","output":"ok"}}"#.to_string(), + token_count("2026-09-01T10:00:12.500Z", 30_050, 30_000, 0), + usage_record("2026-09-01T10:00:20Z", 32_000, 29_952), + compaction, + usage_record("2026-09-01T10:00:30Z", 8_024, 1_024), + r#"{"timestamp":"2026-09-01T10:00:40Z","type":"event_msg","payload":{"type":"task_complete","turn_id":"turn-1"}}"#.to_string(), + ] +} + +#[test] +fn codex_turns_list_their_calls_with_request_starts_and_compaction() { + let scan = scan_lines(&turn_lines( + r#"{"timestamp":"2026-09-01T10:00:21Z","type":"compacted","payload":{"message":"","replacement_history":[]}}"#.to_string(), + )); + + assert_eq!(scan.events.len(), 1); + let event = &scan.events[0]; + assert_eq!( + event.usage.requests, + Some(3), + "the paired token_count is not a call" + ); + let calls = &event.context.as_ref().expect("context").calls; + assert_eq!( + calls + .iter() + .map(|call| (call.requested_at, call.after_compaction)) + .collect::>(), + [ + (at("2026-09-01T10:00:01Z"), false), + (at("2026-09-01T10:00:12Z"), false), + (None, true), + ] + ); + assert_eq!(calls[1].cache_read_tokens, Some(29_952)); + assert_eq!(calls[1].input_tokens, Some(2_048)); + assert_eq!(calls[1].cache_creation_tokens, Some(0)); + assert_eq!(calls[2].completed_at, at("2026-09-01T10:00:30Z").unwrap()); +} + +#[test] +fn an_oversized_compaction_row_still_marks_the_boundary() { + let compaction = format!( + r#"{{"timestamp":"2026-09-01T10:00:21Z","type":"response_item","payload":{{"type":"compaction","encrypted_content":"{}"}}}}"#, + "x".repeat(MAX_JSONL_RECORD_BYTES + 1) + ); + let scan = scan_lines(&turn_lines(compaction)); + + assert_eq!(scan.diagnostics.oversized_rows, 1); + let calls = &scan.events[0].context.as_ref().expect("context").calls; + assert!(calls[2].after_compaction); +} + +#[test] +fn token_counts_alone_do_not_place_request_starts() { + let lines = vec![ + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"session_meta","payload":{"id":"thread-1"}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:01Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"go"}]}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:12Z","type":"response_item","payload":{"type":"function_call_output","call_id":"c1","output":"ok"}}"#.to_string(), + token_count("2026-09-01T10:00:12.500Z", 30_050, 30_000, 0), + r#"{"timestamp":"2026-09-01T10:00:30Z","type":"response_item","payload":{"type":"function_call_output","call_id":"c2","output":"ok"}}"#.to_string(), + token_count("2026-09-01T10:00:30.500Z", 62_150, 32_000, 29_952), + ]; + let scan = scan_lines(&lines); + + assert_eq!(scan.events.len(), 2); + assert!(scan.events.iter().all(|event| event.context.is_none())); +} + +#[test] +fn a_usage_record_outside_a_turn_carries_its_request_start() { + let lines = vec![ + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"session_meta","payload":{"id":"thread-1"}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:01Z","type":"event_msg","payload":{"type":"user_message","message":"go"}}"#.to_string(), + usage_record("2026-09-01T10:00:10Z", 30_000, 0), + ]; + let scan = scan_lines(&lines); + + assert_eq!(scan.events.len(), 1); + let context = scan.events[0].context.as_ref().expect("context"); + assert_eq!(context.requested_at, at("2026-09-01T10:00:01Z")); + assert!(context.calls.is_empty()); +} + +#[test] +fn turn_calls_carry_the_model_that_served_each() { + let scan = scan_lines(&[ + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"session_meta","payload":{"id":"thread-1"}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"event_msg","payload":{"type":"task_started","turn_id":"turn-1"}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"turn_context","payload":{"model":"gpt-5"}}"#.to_string(), + usage_record("2026-09-01T10:00:10Z", 30_000, 0), + r#"{"timestamp":"2026-09-01T10:00:11Z","type":"turn_context","payload":{"model":"gpt-5.1"}}"#.to_string(), + usage_record("2026-09-01T10:00:20Z", 30_000, 0), + r#"{"timestamp":"2026-09-01T10:00:40Z","type":"event_msg","payload":{"type":"task_complete","turn_id":"turn-1"}}"#.to_string(), + ]); + let calls = &scan.events[0].context.as_ref().expect("context").calls; + assert_eq!( + calls + .iter() + .map(|call| call.model.as_deref()) + .collect::>(), + [Some("gpt-5"), Some("gpt-5.1")] + ); + + // A model the rollout never named is a fallback guess, not evidence. + let scan = scan_lines(&turn_lines(String::new())); + let calls = &scan.events[0].context.as_ref().expect("context").calls; + assert!(calls.iter().all(|call| call.model.is_none())); +} + +#[test] +fn completion_usage_keeps_the_turns_compaction() { + let completion = |timestamp: &str| { + format!( + r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"task_complete","completed_at":"{timestamp}","duration_ms":3000}},"usage":{{"input_tokens":30000,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0,"total_tokens":30050}}}}"# + ) + }; + let compacted = r#"{"timestamp":"2026-09-01T10:00:05Z","type":"compacted","payload":{"message":"","replacement_history":[]}}"#; + let opening = [ + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"session_meta","payload":{"id":"thread-1"}}"#, + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"event_msg","payload":{"type":"task_started","turn_id":"turn-1"}}"#, + ] + .map(str::to_string); + + // The completion's own usage is the call after the compaction. + let mut lines = opening.to_vec(); + lines.extend([compacted.to_string(), completion("2026-09-01T10:00:10Z")]); + let scan = scan_lines(&lines); + assert_eq!(scan.events.len(), 1); + assert!( + scan.events[0] + .context + .as_ref() + .expect("context") + .after_compaction + ); + + // A usage line inside the turn took the mark; the completion replaces it. + let mut lines = opening.to_vec(); + lines.extend([ + compacted.to_string(), + usage_record("2026-09-01T10:00:08Z", 30_000, 0), + completion("2026-09-01T10:00:10Z"), + ]); + let scan = scan_lines(&lines); + assert_eq!(scan.events.len(), 1); + assert!( + scan.events[0] + .context + .as_ref() + .expect("context") + .after_compaction + ); +} + +#[test] +fn a_call_without_input_counts_keeps_them_unreported() { + let scan = scan_lines(&[ + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"session_meta","payload":{"id":"thread-1"}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:00Z","type":"event_msg","payload":{"type":"task_started","turn_id":"turn-1"}}"#.to_string(), + usage_record("2026-09-01T10:00:10Z", 30_000, 0), + r#"{"timestamp":"2026-09-01T10:00:20Z","type":"token_usage_record","payload":{"usage":{"output_tokens":50,"total_tokens":50}}}"#.to_string(), + r#"{"timestamp":"2026-09-01T10:00:40Z","type":"event_msg","payload":{"type":"task_complete","turn_id":"turn-1"}}"#.to_string(), + ]); + let event = &scan.events[0]; + let calls = &event.context.as_ref().expect("context").calls; + assert_eq!(calls.len(), 2); + assert_eq!(calls[1].input_tokens, None); + assert_eq!(calls[1].cache_read_tokens, None); + + let analyzed = statsai_core::analyze_cache_calls([event]); + assert_eq!( + analyzed[1].verdict.unclassifiable, + Some(statsai_core::CacheUnclassifiableReason::NoInputTelemetry) + ); +} diff --git a/crates/statsai-adapters/src/codex/tests/parse/mod.rs b/crates/statsai-adapters/src/codex/tests/parse/mod.rs index f7f42f0..708e9cc 100644 --- a/crates/statsai-adapters/src/codex/tests/parse/mod.rs +++ b/crates/statsai-adapters/src/codex/tests/parse/mod.rs @@ -1,5 +1,6 @@ pub(crate) use super::*; +mod calls; mod session; mod usage; diff --git a/crates/statsai-adapters/src/event.rs b/crates/statsai-adapters/src/event.rs index f38a812..6653247 100644 --- a/crates/statsai-adapters/src/event.rs +++ b/crates/statsai-adapters/src/event.rs @@ -402,6 +402,7 @@ pub(crate) fn usage_event( privacy: metadata_only_privacy(), created_at: parts.timestamp, imported_at: Utc::now(), + context: None, } } @@ -491,6 +492,7 @@ pub(crate) fn runtime_to_summary_metrics(runtime: RuntimeInfo) -> SummaryMetrics user_messages: runtime.user_messages, assistant_messages: runtime.assistant_messages, developer_messages: runtime.developer_messages, + cache_health: None, } } diff --git a/crates/statsai-adapters/src/grok/parse/mod.rs b/crates/statsai-adapters/src/grok/parse/mod.rs index d575f54..cac5795 100644 --- a/crates/statsai-adapters/src/grok/parse/mod.rs +++ b/crates/statsai-adapters/src/grok/parse/mod.rs @@ -330,6 +330,7 @@ pub(crate) fn parse_grok_summary( user_messages, assistant_messages, developer_messages: None, + cache_health: None, }); } scan.diagnostics.raw_rows += stats diff --git a/crates/statsai-adapters/src/json.rs b/crates/statsai-adapters/src/json.rs index ee8f0bf..4aeff20 100644 --- a/crates/statsai-adapters/src/json.rs +++ b/crates/statsai-adapters/src/json.rs @@ -8,6 +8,10 @@ use std::io::{BufReader, Cursor}; pub(crate) const MAX_JSONL_RECORD_BYTES: usize = 16 * 1024 * 1024; +/// Leading bytes of an oversized record kept for classification. Enough for +/// the record type and timestamp, which providers write first. +pub(crate) const OVERSIZED_RECORD_HEADER_BYTES: usize = 256; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum BoundedLineRead { Eof, @@ -30,7 +34,7 @@ pub(crate) fn read_bounded_jsonl_line( return Ok(BoundedLineRead::Eof); } if oversized || line.len() > max_bytes { - line.clear(); + line.truncate(oversized_header_len(max_bytes)); return Ok(BoundedLineRead::Oversized); } return Ok(BoundedLineRead::Complete); @@ -62,7 +66,10 @@ pub(crate) fn read_bounded_jsonl_line( if record_bytes <= max_bytes || deferred_cr { line.extend_from_slice(&available[..consumed]); } else { - line.clear(); + let header_len = oversized_header_len(max_bytes); + line.truncate(header_len); + let missing = header_len - line.len(); + line.extend_from_slice(&available[..consumed.min(missing)]); oversized = true; } } @@ -77,6 +84,10 @@ pub(crate) fn read_bounded_jsonl_line( } } +fn oversized_header_len(max_bytes: usize) -> usize { + OVERSIZED_RECORD_HEADER_BYTES.min(max_bytes) +} + pub(crate) fn number_at_any(value: &Value, keys: &[&str]) -> Option { keys.iter() .find_map(|key| value.get(*key).and_then(value_as_u64)) @@ -168,7 +179,7 @@ fn bounded_jsonl_reader_discards_an_oversized_record_and_recovers_next_line() { read_bounded_jsonl_line(&mut reader, &mut line, 8).expect("oversized line"), BoundedLineRead::Oversized ); - assert!(line.is_empty()); + assert_eq!(line, b"xxxxxxxx", "keeps the record's leading bytes"); assert_eq!( read_bounded_jsonl_line(&mut reader, &mut line, 32).expect("next line"), BoundedLineRead::Complete @@ -209,5 +220,5 @@ fn bounded_jsonl_reader_handles_crlf_split_across_buffers_at_the_limit() { .expect("unterminated trailing CR"), BoundedLineRead::Oversized ); - assert!(line.is_empty()); + assert_eq!(line, b"12345678"); } diff --git a/crates/statsai-core/src/cache/detector.rs b/crates/statsai-core/src/cache/detector.rs new file mode 100644 index 0000000..1075597 --- /dev/null +++ b/crates/statsai-core/src/cache/detector.rs @@ -0,0 +1,554 @@ +use super::{ + provider_has_cache_diagnostics, CACHE_LOSS_MIN_PERCENT, CACHE_LOSS_MIN_TOKENS, + CACHE_WRITE_REPORTING_PROVIDERS, +}; +use crate::ids::EventId; +use crate::types::{ModelInfo, UsageEvent}; +use chrono::{DateTime, Utc}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// How the detector classified one model call. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CacheCallClass { + /// The first call observed in its stream. Nothing earlier to compare with. + FirstObserved, + /// The comparison restarts here; see [`CacheBoundaryReason`]. + Boundary, + /// The previous call left too little reusable context to judge a loss. + BelowThreshold, + /// Compared with the previous call in its stream. + Comparable, + /// The call neither read nor wrote the cache, so it did not use it, as + /// when a custom endpoint serves the request. Not judged, and it leaves + /// nothing reusable for the next call. + Uncached, + /// Usage that cannot be placed in order or split into calls. + Unclassifiable, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CacheBoundaryReason { + /// The model changed. Prompt caches are per model. + ModelChange, + /// The provider compacted the conversation, which rewrites the prompt. + Compaction, + /// The previous usage in this stream could not be classified. + EvidenceGap, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CacheUnclassifiableReason { + /// The record had no timestamp, so its place in the stream is unknown. + TimestampMissing, + /// Several calls were recorded as one total with no per-call breakdown. + Aggregated, + /// The provider reported no input or cache counts. + NoInputTelemetry, + /// Another call in the stream has the same timestamp and no recorded order. + AmbiguousOrder, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CacheLoss { + /// No cached reads remained. + Full, + /// Some cached context was still read. + Partial, +} + +/// What a request gap was measured between. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CacheGapTiming { + /// From the previous response to the record that sent this request. + RequestStart, + /// Between the two recorded responses. Includes this call's own run time, + /// so it overstates idle time. + RecordedEvents, +} + +/// The cache lifetime a call's writes asked for, as Claude Code records it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CacheWriteTtl { + FiveMinutes, + OneHour, + Mixed, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct CacheCallVerdict { + pub class: CacheCallClass, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub boundary: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub unclassifiable: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub loss: Option, + /// Reusable context the previous call left, bounded by this call's input. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub baseline_tokens: Option, + /// For a suspected loss, the reusable context it did not read from the + /// cache: the baseline minus its cached reads. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub missed_tokens: Option, + /// What the missed reuse cost at API prices: the call's cost minus its + /// cost had the missed tokens been cache reads. Set by the caller that + /// prices calls; `None` when the model's pricing is unknown. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub missed_cost_micro_usd: Option, + /// Seconds since the previous call in the stream completed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gap_seconds: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gap_timing: Option, + /// The model before a model change. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub previous_model: Option, + /// Cache lifetime the previous call's writes asked for, when recorded. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub previous_write_ttl: Option, +} + +impl CacheCallVerdict { + fn new(class: CacheCallClass) -> Self { + Self { + class, + boundary: None, + unclassifiable: None, + loss: None, + baseline_tokens: None, + missed_tokens: None, + missed_cost_micro_usd: None, + gap_seconds: None, + gap_timing: None, + previous_model: None, + previous_write_ttl: None, + } + } + + fn unclassifiable(reason: CacheUnclassifiableReason) -> Self { + Self { + unclassifiable: Some(reason), + ..Self::new(CacheCallClass::Unclassifiable) + } + } +} + +/// One model call and its verdict, or one unclassifiable block of usage. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct AnalyzedCacheCall { + pub event_id: EventId, + /// Position inside an aggregated event; zero for single-call events. + pub call_index: u32, + /// Model calls this entry stands for: one, or an aggregated event's count. + pub requests: u64, + pub provider: String, + pub source_id: String, + pub provider_account_id: Option, + pub session_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub agent_hash: Option, + pub completed_at: DateTime, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub requested_at: Option>, + pub model: Option, + pub input_tokens: u64, + pub cache_read_tokens: u64, + pub cache_creation_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub write_ttl: Option, + pub verdict: CacheCallVerdict, +} + +impl AnalyzedCacheCall { + /// Ordinary input plus cache writes and reads. + #[must_use] + pub fn logical_input_tokens(&self) -> u64 { + self.input_tokens + .saturating_add(self.cache_creation_tokens.unwrap_or(0)) + .saturating_add(self.cache_read_tokens) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct StreamKey { + provider: String, + source_id: String, + provider_account_id: Option, + session_id: String, + file_hash: Option, + agent_hash: Option, +} + +/// Classifies every model call of the given events. +/// +/// Pass whole sessions: a call is judged against its predecessor, and a +/// predecessor that is not passed in makes the call look like the first one. +/// Events of providers without call order are skipped. The result is sorted by +/// completion time, then event and call position, so equal input always gives +/// equal output. +#[must_use] +pub fn analyze_cache_calls<'a>( + events: impl IntoIterator, +) -> Vec { + let mut streams: BTreeMap> = BTreeMap::new(); + let mut analyzed = Vec::new(); + for event in events { + if !provider_has_cache_diagnostics(&event.provider) { + continue; + } + let calls = event_calls(event); + let timestamp_missing = event + .parse_evidence + .as_ref() + .is_some_and(|evidence| evidence.timestamp_inferred); + if timestamp_missing { + // Its place in the stream is unknown, so it neither gets compared + // nor interrupts the calls around it. + analyzed.push(unclassifiable_entry( + event, + CacheUnclassifiableReason::TimestampMissing, + )); + continue; + } + let line = event + .parse_evidence + .as_ref() + .and_then(|evidence| evidence.source_line_number); + let stream = streams.entry(stream_key(event)).or_default(); + for call in calls { + let order = OrderKey { + completed_at: call.completed_at, + line, + event_id: call.event_id.0.clone(), + call_index: call.call_index, + }; + stream.push((order, call)); + } + } + for mut stream in streams.into_values() { + stream.sort_by(|left, right| left.0.cmp(&right.0)); + mark_ambiguous_order(&mut stream); + classify_stream(&mut stream); + analyzed.extend(stream.into_iter().map(|(_, call)| call)); + } + analyzed.sort_by(|left, right| { + left.completed_at + .cmp(&right.completed_at) + .then_with(|| left.event_id.0.cmp(&right.event_id.0)) + .then_with(|| left.call_index.cmp(&right.call_index)) + }); + analyzed +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct OrderKey { + completed_at: DateTime, + line: Option, + event_id: String, + call_index: u32, +} + +fn stream_key(event: &UsageEvent) -> StreamKey { + StreamKey { + provider: event.provider.clone(), + source_id: event.source_id.0.clone(), + provider_account_id: event.provider_account_id.as_ref().map(|id| id.0.clone()), + session_id: event.session.session_id.clone(), + file_hash: event + .parse_evidence + .as_ref() + .and_then(|evidence| evidence.source_file_path_hash.clone()), + agent_hash: event + .context + .as_ref() + .and_then(|context| context.agent_hash.clone()), + } +} + +/// The name the detector compares to tell whether consecutive calls used the +/// same model, and so the same prompt cache. +#[must_use] +pub fn cache_model_name(model: &ModelInfo) -> Option { + model + .normalized_name + .clone() + .or_else(|| model.name.clone()) + .filter(|name| !name.trim().is_empty()) +} + +/// The event's model, unless the parser filled in a fallback because the +/// transcript did not name one: a guess cannot show that the model changed. +fn event_model(event: &UsageEvent) -> Option { + let inferred = event + .parse_evidence + .as_ref() + .is_some_and(|evidence| evidence.model_inferred); + event + .model + .as_ref() + .filter(|_| !inferred) + .and_then(cache_model_name) +} + +fn base_entry(event: &UsageEvent, verdict: CacheCallVerdict) -> AnalyzedCacheCall { + AnalyzedCacheCall { + event_id: event.event_id.clone(), + call_index: 0, + requests: 1, + provider: event.provider.clone(), + source_id: event.source_id.0.clone(), + provider_account_id: event.provider_account_id.as_ref().map(|id| id.0.clone()), + session_id: event.session.session_id.clone(), + agent_hash: event + .context + .as_ref() + .and_then(|context| context.agent_hash.clone()), + completed_at: event.created_at, + requested_at: None, + model: event_model(event), + input_tokens: event.usage.input_tokens.unwrap_or(0), + cache_read_tokens: event.usage.cache_read_tokens.unwrap_or(0), + cache_creation_tokens: event.usage.cache_creation_tokens, + write_ttl: None, + verdict, + } +} + +fn unclassifiable_entry( + event: &UsageEvent, + reason: CacheUnclassifiableReason, +) -> AnalyzedCacheCall { + AnalyzedCacheCall { + requests: event.usage.requests.unwrap_or(1).max(1), + ..base_entry(event, CacheCallVerdict::unclassifiable(reason)) + } +} + +/// The calls an event stands for, before classification. An event that cannot +/// be split into calls comes back as one unclassifiable entry. +fn event_calls(event: &UsageEvent) -> Vec { + let requests = event.usage.requests.unwrap_or(1).max(1); + let recorded_calls = event + .context + .as_ref() + .map(|context| context.calls.as_slice()) + .unwrap_or_default(); + if !recorded_calls.is_empty() { + if recorded_calls.len() as u64 != requests { + return vec![unclassifiable_entry( + event, + CacheUnclassifiableReason::Aggregated, + )]; + } + return recorded_calls + .iter() + .enumerate() + .map(|(index, call)| { + let verdict = if call.input_tokens.is_none() && call.cache_read_tokens.is_none() { + CacheCallVerdict::unclassifiable(CacheUnclassifiableReason::NoInputTelemetry) + } else { + CacheCallVerdict { + boundary: call + .after_compaction + .then_some(CacheBoundaryReason::Compaction), + ..CacheCallVerdict::new(CacheCallClass::Comparable) + } + }; + AnalyzedCacheCall { + call_index: index as u32, + completed_at: call.completed_at, + requested_at: call.requested_at, + input_tokens: call.input_tokens.unwrap_or(0), + cache_read_tokens: call.cache_read_tokens.unwrap_or(0), + cache_creation_tokens: call.cache_creation_tokens, + // A turn's calls can switch models, so the event's model, + // which is the turn's last, does not stand in for each. + model: call.model.clone(), + verdict, + ..base_entry(event, CacheCallVerdict::new(CacheCallClass::Comparable)) + } + }) + .collect(); + } + if requests > 1 { + return vec![unclassifiable_entry( + event, + CacheUnclassifiableReason::Aggregated, + )]; + } + if event.usage.input_tokens.is_none() && event.usage.cache_read_tokens.is_none() { + return vec![unclassifiable_entry( + event, + CacheUnclassifiableReason::NoInputTelemetry, + )]; + } + let context = event.context.as_ref(); + let compaction = context.is_some_and(|context| context.after_compaction); + vec![AnalyzedCacheCall { + requested_at: context.and_then(|context| context.requested_at), + write_ttl: write_ttl( + event.usage.cache_creation_5m_tokens, + event.usage.cache_creation_1h_tokens, + ), + verdict: CacheCallVerdict { + boundary: compaction.then_some(CacheBoundaryReason::Compaction), + ..CacheCallVerdict::new(CacheCallClass::Comparable) + }, + ..base_entry(event, CacheCallVerdict::new(CacheCallClass::Comparable)) + }] +} + +fn write_ttl(five_minutes: Option, one_hour: Option) -> Option { + match (five_minutes.unwrap_or(0) > 0, one_hour.unwrap_or(0) > 0) { + (true, true) => Some(CacheWriteTtl::Mixed), + (true, false) => Some(CacheWriteTtl::FiveMinutes), + (false, true) => Some(CacheWriteTtl::OneHour), + (false, false) => None, + } +} + +/// Calls with one timestamp keep their order only when every two of them carry +/// source lines or belong to the same event. Otherwise none of them is +/// compared: any could have come first, so no member's predecessor is known. +fn mark_ambiguous_order(stream: &mut [(OrderKey, AnalyzedCacheCall)]) { + let mut start = 0; + while start < stream.len() { + let completed_at = stream[start].0.completed_at; + let end = start + + stream[start..] + .iter() + .take_while(|(order, _)| order.completed_at == completed_at) + .count(); + let group = &mut stream[start..end]; + let ordered = group.iter().enumerate().all(|(index, (left, _))| { + group[index + 1..].iter().all(|(right, _)| { + left.event_id == right.event_id || (left.line.is_some() && right.line.is_some()) + }) + }); + if !ordered { + for (_, call) in group.iter_mut() { + if call.verdict.class != CacheCallClass::Unclassifiable { + call.verdict = + CacheCallVerdict::unclassifiable(CacheUnclassifiableReason::AmbiguousOrder); + } + } + } + start = end; + } +} + +struct Previous { + completed_at: DateTime, + reusable_tokens: u64, + model: Option, + write_ttl: Option, +} + +fn classify_stream(stream: &mut [(OrderKey, AnalyzedCacheCall)]) { + let mut started = false; + let mut after_gap = false; + let mut previous: Option = None; + for (_, call) in stream.iter_mut() { + if call.verdict.class == CacheCallClass::Unclassifiable { + started = true; + after_gap = true; + previous = None; + continue; + } + let compaction = call.verdict.boundary == Some(CacheBoundaryReason::Compaction); + let mut verdict = CacheCallVerdict::new(CacheCallClass::Comparable); + if let Some(previous) = previous.as_ref() { + let (gap_seconds, gap_timing) = gap(previous.completed_at, call); + verdict.gap_seconds = Some(gap_seconds); + verdict.gap_timing = Some(gap_timing); + verdict.previous_write_ttl = previous.write_ttl; + } + let model_changed = previous.as_ref().is_some_and(|previous| { + previous.model.is_some() && call.model.is_some() && previous.model != call.model + }); + if skipped_the_cache(call) { + verdict.class = CacheCallClass::Uncached; + } else if !started { + verdict.class = CacheCallClass::FirstObserved; + } else if after_gap { + verdict.class = CacheCallClass::Boundary; + verdict.boundary = Some(CacheBoundaryReason::EvidenceGap); + } else if compaction { + verdict.class = CacheCallClass::Boundary; + verdict.boundary = Some(CacheBoundaryReason::Compaction); + } else if model_changed { + verdict.class = CacheCallClass::Boundary; + verdict.boundary = Some(CacheBoundaryReason::ModelChange); + verdict.previous_model = previous + .as_ref() + .and_then(|previous| previous.model.clone()); + } else if let Some(previous) = previous.as_ref() { + let baseline = previous.reusable_tokens.min(call.logical_input_tokens()); + verdict.baseline_tokens = Some(baseline); + if baseline < CACHE_LOSS_MIN_TOKENS { + verdict.class = CacheCallClass::BelowThreshold; + } else { + verdict.loss = loss(baseline, call.cache_read_tokens); + verdict.missed_tokens = verdict + .loss + .map(|_| baseline.saturating_sub(call.cache_read_tokens)); + } + } + call.verdict = verdict; + started = true; + after_gap = false; + previous = Some(Previous { + completed_at: call.completed_at, + reusable_tokens: call + .cache_read_tokens + .saturating_add(call.cache_creation_tokens.unwrap_or(0)), + model: call.model.clone(), + write_ttl: call.write_ttl, + }); + } +} + +/// Whether a call reported no cache reads and no cache writes, for a provider +/// whose calls report writes whenever they use the cache. +fn skipped_the_cache(call: &AnalyzedCacheCall) -> bool { + CACHE_WRITE_REPORTING_PROVIDERS.contains(&call.provider.as_str()) + && call.input_tokens > 0 + && call.cache_read_tokens == 0 + && call.cache_creation_tokens == Some(0) +} + +/// Seconds from the previous response to this request. A recorded request +/// start that precedes the previous response belongs to an earlier exchange, +/// so the gap falls back to the two responses' own timestamps. +fn gap(previous_completed_at: DateTime, call: &AnalyzedCacheCall) -> (u64, CacheGapTiming) { + let seconds = |from: DateTime, to: DateTime| (to - from).num_seconds().max(0) as u64; + match call.requested_at { + Some(requested_at) if requested_at >= previous_completed_at => ( + seconds(previous_completed_at, requested_at), + CacheGapTiming::RequestStart, + ), + _ => ( + seconds(previous_completed_at, call.completed_at), + CacheGapTiming::RecordedEvents, + ), + } +} + +fn loss(baseline: u64, cache_read_tokens: u64) -> Option { + let drop = baseline.saturating_sub(cache_read_tokens); + let suspected = drop >= CACHE_LOSS_MIN_TOKENS + && u128::from(drop) * 100 >= u128::from(baseline) * u128::from(CACHE_LOSS_MIN_PERCENT); + suspected.then_some(if cache_read_tokens == 0 { + CacheLoss::Full + } else { + CacheLoss::Partial + }) +} diff --git a/crates/statsai-core/src/cache/health.rs b/crates/statsai-core/src/cache/health.rs new file mode 100644 index 0000000..779019f --- /dev/null +++ b/crates/statsai-core/src/cache/health.rs @@ -0,0 +1,246 @@ +use super::{ + cache_gap_bin, AnalyzedCacheCall, CacheBoundaryReason, CacheCallClass, CacheGapTiming, + CacheLoss, CACHE_GAP_BIN_COUNT, CACHE_HEALTH_VERSION, +}; +use crate::types::UsageEvent; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// Bounded prompt-cache diagnostics for one daily summary. +/// +/// Only counts leave the device: no timestamps, ids, or per-call values. Token +/// totals stay on the summary's `usage`, so this adds only what they cannot +/// say. `requests` equals `unclassifiable + first_observed + boundaries + +/// below_threshold + uncached + comparable`, and the gap bins add up to +/// `comparable`, `losses`, and `missed_tokens`. Monthly rollups embed every +/// day's object, so the fields most days leave at zero are left out then. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct CacheHealthV1 { + pub version: u32, + /// Model calls of providers the detector covers. + pub requests: u64, + pub unclassifiable: u64, + /// First calls of their stream. + pub first_observed: u64, + /// First calls that read nothing from the cache. + pub first_cold: u64, + /// Calls after a model change, compaction, or unclassifiable usage. + pub boundaries: u64, + /// Calls whose predecessor left too little reusable context to judge. + pub below_threshold: u64, + /// Calls that neither read nor wrote the cache, so did not use it. + #[serde(default, skip_serializing_if = "is_zero")] + pub uncached: u64, + pub comparable: u64, + /// Suspected cache losses among comparable calls. + pub losses: u64, + /// Suspected losses that read nothing from the cache. + pub full_losses: u64, + /// Reusable context suspected losses did not read from the cache. + #[serde(default, skip_serializing_if = "is_zero")] + pub missed_tokens: u64, + /// What that missed reuse cost at API prices, in micro-USD. + #[serde(default, skip_serializing_if = "is_zero")] + pub missed_cost_micro_usd: u64, + /// Suspected losses on models without known pricing, left out of + /// `missed_cost_micro_usd`. + #[serde(default, skip_serializing_if = "is_zero")] + pub unpriced_losses: u64, + /// Comparable calls whose gap was measured between recorded responses + /// rather than from a recorded request start. + pub timing_estimated: u64, + /// Ordinary input from calls whose provider did not report cache writes, + /// so it may include writes. + pub write_split_unavailable_input_tokens: u64, + /// Comparable calls by gap since the previous call, one count per bin of + /// [`super::CACHE_GAP_BIN_UPPER_MINUTES`]: `<1m` first, `60m+` last. + pub gap_comparable: Vec, + /// Suspected losses in the same bins. + pub gap_losses: Vec, + /// Missed tokens in the same bins. + #[serde(default = "empty_gap_bins", skip_serializing_if = "all_zero")] + pub gap_missed_tokens: Vec, +} + +fn is_zero(value: &u64) -> bool { + *value == 0 +} + +fn all_zero(values: &[u64]) -> bool { + values.iter().all(|value| *value == 0) +} + +fn empty_gap_bins() -> Vec { + vec![0; CACHE_GAP_BIN_COUNT] +} + +impl Default for CacheHealthV1 { + fn default() -> Self { + Self { + version: CACHE_HEALTH_VERSION, + requests: 0, + unclassifiable: 0, + first_observed: 0, + first_cold: 0, + boundaries: 0, + below_threshold: 0, + uncached: 0, + comparable: 0, + losses: 0, + full_losses: 0, + missed_tokens: 0, + missed_cost_micro_usd: 0, + unpriced_losses: 0, + timing_estimated: 0, + write_split_unavailable_input_tokens: 0, + gap_comparable: empty_gap_bins(), + gap_losses: empty_gap_bins(), + gap_missed_tokens: empty_gap_bins(), + } + } +} + +impl CacheHealthV1 { + /// Calls the detector could place and classify. + #[must_use] + pub fn analyzed(&self) -> u64 { + self.requests.saturating_sub(self.unclassifiable) + } + + pub fn add_call(&mut self, call: &AnalyzedCacheCall) { + self.requests = self.requests.saturating_add(call.requests); + let verdict = &call.verdict; + match verdict.class { + CacheCallClass::Unclassifiable => { + self.unclassifiable = self.unclassifiable.saturating_add(call.requests); + } + CacheCallClass::FirstObserved => { + self.first_observed = self.first_observed.saturating_add(1); + if call.cache_read_tokens == 0 { + self.first_cold = self.first_cold.saturating_add(1); + } + } + CacheCallClass::Boundary => self.boundaries = self.boundaries.saturating_add(1), + CacheCallClass::BelowThreshold => { + self.below_threshold = self.below_threshold.saturating_add(1); + } + CacheCallClass::Uncached => self.uncached = self.uncached.saturating_add(1), + CacheCallClass::Comparable => { + self.comparable = self.comparable.saturating_add(1); + if verdict.gap_timing == Some(CacheGapTiming::RecordedEvents) { + self.timing_estimated = self.timing_estimated.saturating_add(1); + } + let lost = verdict.loss.is_some(); + if lost { + self.losses = self.losses.saturating_add(1); + } + if verdict.loss == Some(CacheLoss::Full) { + self.full_losses = self.full_losses.saturating_add(1); + } + match (lost, verdict.missed_cost_micro_usd) { + (true, Some(cost)) => { + self.missed_cost_micro_usd = + self.missed_cost_micro_usd.saturating_add(cost); + } + (true, None) => self.unpriced_losses = self.unpriced_losses.saturating_add(1), + (false, _) => {} + } + let missed = verdict.missed_tokens.unwrap_or(0); + self.missed_tokens = self.missed_tokens.saturating_add(missed); + if let Some(bin) = verdict.gap_seconds.map(cache_gap_bin) { + if let Some(count) = self.gap_comparable.get_mut(bin) { + *count = count.saturating_add(1); + } + if let Some(count) = self.gap_losses.get_mut(bin).filter(|_| lost) { + *count = count.saturating_add(1); + } + if let Some(tokens) = self.gap_missed_tokens.get_mut(bin) { + *tokens = tokens.saturating_add(missed); + } + } + } + } + } + + /// Counts ordinary input as unsplit when its provider reported no cache + /// writes. An event that lists its calls is counted call by call: summing + /// one reported zero with unreported writes would hide the rest. + pub fn add_event_input(&mut self, event: &UsageEvent) { + let calls = event + .context + .as_ref() + .map(|context| context.calls.as_slice()) + .unwrap_or_default(); + let unsplit = if !calls.is_empty() { + calls + .iter() + .filter(|call| call.cache_creation_tokens.is_none()) + .map(|call| call.input_tokens.unwrap_or(0)) + .fold(0, u64::saturating_add) + } else if event.usage.cache_creation_tokens.is_none() { + event.usage.input_tokens.unwrap_or(0) + } else { + 0 + }; + self.write_split_unavailable_input_tokens = self + .write_split_unavailable_input_tokens + .saturating_add(unsplit); + } + + pub fn merge(&mut self, other: &Self) { + self.requests = self.requests.saturating_add(other.requests); + self.unclassifiable = self.unclassifiable.saturating_add(other.unclassifiable); + self.first_observed = self.first_observed.saturating_add(other.first_observed); + self.first_cold = self.first_cold.saturating_add(other.first_cold); + self.boundaries = self.boundaries.saturating_add(other.boundaries); + self.below_threshold = self.below_threshold.saturating_add(other.below_threshold); + self.uncached = self.uncached.saturating_add(other.uncached); + self.comparable = self.comparable.saturating_add(other.comparable); + self.losses = self.losses.saturating_add(other.losses); + self.full_losses = self.full_losses.saturating_add(other.full_losses); + self.missed_tokens = self.missed_tokens.saturating_add(other.missed_tokens); + self.missed_cost_micro_usd = self + .missed_cost_micro_usd + .saturating_add(other.missed_cost_micro_usd); + self.unpriced_losses = self.unpriced_losses.saturating_add(other.unpriced_losses); + self.timing_estimated = self.timing_estimated.saturating_add(other.timing_estimated); + self.write_split_unavailable_input_tokens = self + .write_split_unavailable_input_tokens + .saturating_add(other.write_split_unavailable_input_tokens); + for (count, other) in self.gap_comparable.iter_mut().zip(&other.gap_comparable) { + *count = count.saturating_add(*other); + } + for (count, other) in self.gap_losses.iter_mut().zip(&other.gap_losses) { + *count = count.saturating_add(*other); + } + for (tokens, other) in self + .gap_missed_tokens + .iter_mut() + .zip(&other.gap_missed_tokens) + { + *tokens = tokens.saturating_add(*other); + } + } +} + +/// Boundary counts by reason. Local only: the synced object carries the total. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct CacheBoundaryCounts { + pub model_change: u64, + pub compaction: u64, + pub evidence_gap: u64, +} + +impl CacheBoundaryCounts { + pub fn add_call(&mut self, call: &AnalyzedCacheCall) { + if call.verdict.class != CacheCallClass::Boundary { + return; + } + let count = match call.verdict.boundary { + Some(CacheBoundaryReason::ModelChange) => &mut self.model_change, + Some(CacheBoundaryReason::Compaction) => &mut self.compaction, + Some(CacheBoundaryReason::EvidenceGap) | None => &mut self.evidence_gap, + }; + *count = count.saturating_add(1); + } +} diff --git a/crates/statsai-core/src/cache/mod.rs b/crates/statsai-core/src/cache/mod.rs new file mode 100644 index 0000000..e416d10 --- /dev/null +++ b/crates/statsai-core/src/cache/mod.rs @@ -0,0 +1,97 @@ +//! Prompt-cache diagnostics. +//! +//! The report answers three questions for a period: how much logical input was +//! served from the provider's prompt cache, when that reuse dropped, and how +//! much context the model calls processed regardless. The first and last come +//! straight from token totals. The second needs request order, so it runs a +//! conservative detector over each stream of model calls. +//! +//! A stream is one agent's calls within one session: the same provider, source, +//! account, session, transcript file, and sub-agent. Each call is compared only +//! with the call before it in its stream. The previous call leaves its cache +//! reads plus its known cache writes reusable; the current call is expected to +//! read about that much again, bounded by its own input size. When reads fall at +//! least half and at least [`CACHE_LOSS_MIN_TOKENS`] below that baseline, the +//! call is a suspected cache loss. That is a heuristic over recorded token +//! counts, not a provider-confirmed miss. +//! +//! Comparisons restart, rather than count as losses, at the first call of a +//! stream, after a model change, after compaction, and after a call whose +//! usage cannot be placed or split into calls. A call that neither read nor +//! wrote the cache did not use it, as when a custom endpoint serves the +//! request, so it is counted apart and not judged. + +mod detector; +mod health; +mod report; + +pub use detector::*; +pub use health::*; +pub use report::*; + +/// Version of the detector rules and of the synced `cache_health` object. +pub const CACHE_HEALTH_VERSION: u32 = 1; + +/// Smallest drop in cached reads that can count as a loss. +pub const CACHE_LOSS_MIN_TOKENS: u64 = 4_096; + +/// A loss also needs reads to fall by at least this share of the baseline. +pub const CACHE_LOSS_MIN_PERCENT: u64 = 50; + +/// Upper bounds, in minutes, of the request-gap bins. The last bin is open. +pub const CACHE_GAP_BIN_UPPER_MINUTES: [u64; 7] = [1, 5, 10, 15, 30, 45, 60]; + +/// Number of request-gap bins, including the open-ended last one. +pub const CACHE_GAP_BIN_COUNT: usize = CACHE_GAP_BIN_UPPER_MINUTES.len() + 1; + +/// Providers whose usage events record calls in order, which the detector +/// needs. Token totals for every other provider still count toward hit rates. +pub const CACHE_DIAGNOSTIC_PROVIDERS: [&str; 2] = ["claude_code", "codex"]; + +#[must_use] +pub fn provider_has_cache_diagnostics(provider: &str) -> bool { + CACHE_DIAGNOSTIC_PROVIDERS.contains(&provider) +} + +/// Providers that report cache writes on every call that uses the cache, so a +/// call reporting no reads and no writes did not use it. Codex reports zero +/// writes on calls that do, so it is not one of them. +pub const CACHE_WRITE_REPORTING_PROVIDERS: [&str; 1] = ["claude_code"]; + +/// Index of the gap bin for a gap between calls. Bins are closed below and +/// open above, so a gap of exactly five minutes lands in `5–10`. +#[must_use] +pub fn cache_gap_bin(gap_seconds: u64) -> usize { + CACHE_GAP_BIN_UPPER_MINUTES + .iter() + .position(|upper| gap_seconds < upper * 60) + .unwrap_or(CACHE_GAP_BIN_COUNT - 1) +} + +/// Label of a gap bin, such as `<1m`, `5–10m`, or `60m+`. +#[must_use] +pub fn cache_gap_bin_label(index: usize) -> String { + let lower = index + .checked_sub(1) + .and_then(|previous| CACHE_GAP_BIN_UPPER_MINUTES.get(previous)); + match (lower, CACHE_GAP_BIN_UPPER_MINUTES.get(index)) { + (None, Some(upper)) => format!("<{upper}m"), + (Some(lower), Some(upper)) => format!("{lower}–{upper}m"), + (Some(lower), None) => format!("{lower}m+"), + (None, None) => String::new(), + } +} + +/// Cached reads as a share of all logical input: ordinary input, cache writes, +/// and cache reads. Token-weighted, so a long call counts for its size. +#[must_use] +pub fn cache_hit_ratio( + input_tokens: u64, + cache_creation_tokens: u64, + cache_read_tokens: u64, +) -> Option { + let logical = input_tokens + .saturating_add(cache_creation_tokens) + .saturating_add(cache_read_tokens); + (logical > 0).then(|| cache_read_tokens as f64 / logical as f64) +} diff --git a/crates/statsai-core/src/cache/report.rs b/crates/statsai-core/src/cache/report.rs new file mode 100644 index 0000000..dce04bb --- /dev/null +++ b/crates/statsai-core/src/cache/report.rs @@ -0,0 +1,346 @@ +use super::{ + cache_gap_bin_label, cache_hit_ratio, provider_has_cache_diagnostics, AnalyzedCacheCall, + CacheBoundaryCounts, CacheCallClass, CacheHealthV1, CACHE_GAP_BIN_UPPER_MINUTES, + CACHE_HEALTH_VERSION, +}; +use crate::types::{UsageCounts, UsageEvent, UsageSummary}; +use chrono::{DateTime, Duration, Offset, TimeZone, Utc}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +pub const CACHE_REPORT_SCHEMA_VERSION: &str = "cache_report.v1"; + +/// Width of a `--timeline` bucket. +pub const CACHE_TIMELINE_BUCKET_MINUTES: i64 = 10; + +/// The prompt-cache report for a period, as `statsai report cache --json` and +/// the local API print it. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct CacheReport { + pub schema_version: String, + pub detector_version: u32, + pub label: String, + pub since: Option>, + pub until: DateTime, + pub filters: CacheReportFilters, + /// Token totals across every provider in scope. + pub totals: CacheTokenTotals, + /// Detector counts across the providers it covers. + pub diagnostics: CacheHealthV1, + pub boundaries: CacheBoundaryCounts, + pub gap_histogram: Vec, + pub providers: Vec, + /// UTC days, matching the daily summaries that sync. + pub days: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timeline: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub details: Option>, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct CacheReportFilters { + pub provider: Option, + pub account: Option, + pub session: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct CacheTokenTotals { + pub requests: u64, + pub input_tokens: u64, + pub cache_creation_tokens: u64, + pub cache_read_tokens: u64, + /// Ordinary input, cache writes, and cache reads: the context processed. + pub logical_input_tokens: u64, + /// Cache reads over logical input. + pub cache_hit_ratio: Option, + pub avg_logical_input_per_request: Option, +} + +impl CacheTokenTotals { + pub fn add_event(&mut self, event: &UsageEvent) { + self.add_usage(&event.usage, 1); + } + + /// Adds a summary that has no events behind it, such as a provider's own + /// session totals. It counts toward token totals, never diagnostics. + pub fn add_summary(&mut self, summary: &UsageSummary) { + self.add_usage(&summary.usage, 0); + } + + fn add_usage(&mut self, usage: &UsageCounts, default_requests: u64) { + self.requests = self + .requests + .saturating_add(usage.requests.unwrap_or(default_requests)); + self.input_tokens = self + .input_tokens + .saturating_add(usage.input_tokens.unwrap_or(0)); + self.cache_creation_tokens = self + .cache_creation_tokens + .saturating_add(usage.cache_creation_tokens.unwrap_or(0)); + self.cache_read_tokens = self + .cache_read_tokens + .saturating_add(usage.cache_read_tokens.unwrap_or(0)); + self.finish(); + } + + fn finish(&mut self) { + self.logical_input_tokens = self + .input_tokens + .saturating_add(self.cache_creation_tokens) + .saturating_add(self.cache_read_tokens); + self.cache_hit_ratio = cache_hit_ratio( + self.input_tokens, + self.cache_creation_tokens, + self.cache_read_tokens, + ); + self.avg_logical_input_per_request = + (self.requests > 0).then(|| self.logical_input_tokens as f64 / self.requests as f64); + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct CacheGapRow { + pub label: String, + pub min_minutes: u64, + /// Exclusive upper bound; `None` for the open last bin. + pub max_minutes: Option, + pub comparable: u64, + pub losses: u64, + /// Reusable context the bin's suspected losses did not read from the cache. + pub missed_tokens: u64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct CacheProviderRow { + pub provider: String, + pub totals: CacheTokenTotals, + /// `None` for providers the detector does not cover. + pub diagnostics: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct CacheDayRow { + pub day: String, + pub totals: CacheTokenTotals, + pub diagnostics: Option, +} + +/// One local-time bucket of model calls, keyed by when each call completed. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct CacheTimelineRow { + /// Bucket start in local time, RFC 3339 with offset. + pub start: String, + pub requests: u64, + pub logical_input_tokens: u64, + pub cache_read_tokens: u64, + pub cache_hit_ratio: Option, + pub first_observed: u64, + pub boundaries: u64, + pub losses: u64, + pub missed_tokens: u64, +} + +/// Inputs for [`build_cache_report`]. `events` are the events in scope; +/// `calls` are their analyzed calls, judged against whole sessions. +/// `summaries` are single-day summaries in scope with no events behind them, +/// which sync with the daily summaries and count toward the same totals. +pub struct CacheReportInput<'a> { + pub label: String, + pub since: Option>, + pub until: DateTime, + pub filters: CacheReportFilters, + pub events: &'a [UsageEvent], + pub summaries: &'a [UsageSummary], + pub calls: Vec, + pub include_details: bool, +} + +#[must_use] +pub fn build_cache_report( + input: CacheReportInput<'_>, + timeline: Option<&Tz>, +) -> CacheReport +where + Tz::Offset: std::fmt::Display, +{ + let mut totals = CacheTokenTotals::default(); + let mut diagnostics = CacheHealthV1::default(); + let mut boundaries = CacheBoundaryCounts::default(); + let mut providers: BTreeMap)> = + BTreeMap::new(); + let mut days: BTreeMap)> = BTreeMap::new(); + let mut event_days: BTreeMap<&str, String> = BTreeMap::new(); + + for event in input.events { + let supported = provider_has_cache_diagnostics(&event.provider); + let day = event.session.started_at.date_naive().to_string(); + event_days.insert(event.event_id.0.as_str(), day.clone()); + totals.add_event(event); + for (totals, health) in [ + providers.entry(event.provider.clone()).or_default(), + days.entry(day).or_default(), + ] { + totals.add_event(event); + if supported { + health + .get_or_insert_with(CacheHealthV1::default) + .add_event_input(event); + } + } + if supported { + diagnostics.add_event_input(event); + } + } + for summary in input.summaries { + let day = summary + .period_start + .unwrap_or(summary.observed_at) + .date_naive() + .to_string(); + totals.add_summary(summary); + providers + .entry(summary.provider.clone()) + .or_default() + .0 + .add_summary(summary); + days.entry(day).or_default().0.add_summary(summary); + } + for call in &input.calls { + diagnostics.add_call(call); + boundaries.add_call(call); + if let Some((_, health)) = providers.get_mut(&call.provider) { + health + .get_or_insert_with(CacheHealthV1::default) + .add_call(call); + } + if let Some((_, health)) = event_days + .get(call.event_id.0.as_str()) + .and_then(|day| days.get_mut(day)) + { + health + .get_or_insert_with(CacheHealthV1::default) + .add_call(call); + } + } + + CacheReport { + schema_version: CACHE_REPORT_SCHEMA_VERSION.to_string(), + detector_version: CACHE_HEALTH_VERSION, + label: input.label, + since: input.since, + until: input.until, + filters: input.filters, + totals, + gap_histogram: gap_histogram(&diagnostics), + diagnostics, + boundaries, + providers: providers + .into_iter() + .map(|(provider, (totals, diagnostics))| CacheProviderRow { + provider, + totals, + diagnostics, + }) + .collect(), + days: days + .into_iter() + .map(|(day, (totals, diagnostics))| CacheDayRow { + day, + totals, + diagnostics, + }) + .collect(), + timeline: timeline.map(|tz| cache_timeline(&input.calls, tz)), + details: input.include_details.then_some(input.calls), + } +} + +#[must_use] +pub fn gap_histogram(health: &CacheHealthV1) -> Vec { + health + .gap_comparable + .iter() + .zip(&health.gap_losses) + .zip(&health.gap_missed_tokens) + .enumerate() + .map( + |(index, ((comparable, losses), missed_tokens))| CacheGapRow { + label: cache_gap_bin_label(index), + min_minutes: index + .checked_sub(1) + .and_then(|previous| CACHE_GAP_BIN_UPPER_MINUTES.get(previous)) + .copied() + .unwrap_or(0), + max_minutes: CACHE_GAP_BIN_UPPER_MINUTES.get(index).copied(), + comparable: *comparable, + losses: *losses, + missed_tokens: *missed_tokens, + }, + ) + .collect() +} + +/// Groups calls into [`CACHE_TIMELINE_BUCKET_MINUTES`] buckets of local time. +/// Buckets without calls are left out. +#[must_use] +pub fn cache_timeline(calls: &[AnalyzedCacheCall], tz: &Tz) -> Vec +where + Tz::Offset: std::fmt::Display, +{ + let width = Duration::minutes(CACHE_TIMELINE_BUCKET_MINUTES).num_seconds(); + let mut buckets: BTreeMap, CacheTimelineRow> = BTreeMap::new(); + for call in calls { + // Truncating local wall-clock seconds keeps buckets on local ten-minute + // marks in zones whose offset is not a whole number of hours. + let offset = i64::from( + call.completed_at + .with_timezone(tz) + .offset() + .fix() + .local_minus_utc(), + ); + let local_seconds = call.completed_at.timestamp() + offset; + let start_seconds = local_seconds - local_seconds.rem_euclid(width) - offset; + let Some(start) = DateTime::::from_timestamp(start_seconds, 0) else { + continue; + }; + let row = buckets.entry(start).or_insert_with(|| CacheTimelineRow { + start: start.with_timezone(tz).to_rfc3339(), + requests: 0, + logical_input_tokens: 0, + cache_read_tokens: 0, + cache_hit_ratio: None, + first_observed: 0, + boundaries: 0, + losses: 0, + missed_tokens: 0, + }); + row.requests = row.requests.saturating_add(call.requests); + row.logical_input_tokens = row + .logical_input_tokens + .saturating_add(call.logical_input_tokens()); + row.cache_read_tokens = row.cache_read_tokens.saturating_add(call.cache_read_tokens); + match call.verdict.class { + CacheCallClass::FirstObserved => row.first_observed += 1, + CacheCallClass::Boundary => row.boundaries += 1, + _ => {} + } + if call.verdict.loss.is_some() { + row.losses += 1; + } + row.missed_tokens = row + .missed_tokens + .saturating_add(call.verdict.missed_tokens.unwrap_or(0)); + } + buckets + .into_values() + .map(|mut row| { + row.cache_hit_ratio = (row.logical_input_tokens > 0) + .then(|| row.cache_read_tokens as f64 / row.logical_input_tokens as f64); + row + }) + .collect() +} diff --git a/crates/statsai-core/src/lib.rs b/crates/statsai-core/src/lib.rs index 6292f72..8f2e9eb 100644 --- a/crates/statsai-core/src/lib.rs +++ b/crates/statsai-core/src/lib.rs @@ -2,6 +2,7 @@ mod account_plan; mod archive; +mod cache; mod code_changes; mod daemon_presence; mod ids; @@ -14,6 +15,7 @@ mod types; pub use account_plan::*; pub use archive::*; +pub use cache::*; pub use code_changes::*; pub use daemon_presence::*; pub use ids::*; diff --git a/crates/statsai-core/src/tests/cache.rs b/crates/statsai-core/src/tests/cache.rs new file mode 100644 index 0000000..6fd1230 --- /dev/null +++ b/crates/statsai-core/src/tests/cache.rs @@ -0,0 +1,849 @@ +use super::support::{test_event, test_source}; +use super::*; +use chrono::{DateTime, Duration, FixedOffset, TimeZone, Utc}; + +fn at(seconds: i64) -> DateTime { + Utc.with_ymd_and_hms(2026, 3, 2, 10, 0, 0).unwrap() + Duration::seconds(seconds) +} + +/// One single-call event in session `s`, transcript `main`, at `seconds`. +fn call(index: u64, seconds: i64, input: u64, read: u64, write: Option) -> UsageEvent { + let source = test_source("claude_code", "/fixture/claude"); + let mut event = test_event("claude_code", &source, at(seconds), 0, None); + event.event_id = EventId(format!("event_{index:03}")); + event.model = Some(ModelInfo { + normalized_name: Some("claude-model".to_string()), + ..ModelInfo::default() + }); + event.usage = UsageCounts { + input_tokens: Some(input), + cache_read_tokens: Some(read), + cache_creation_tokens: write, + output_tokens: Some(10), + requests: Some(1), + ..UsageCounts::default() + }; + event.parse_evidence = Some(ParseEvidence { + event_key_version: "test".to_string(), + source_file_path_hash: Some("main".to_string()), + source_line_number: Some(index), + source_record_id: None, + model_inferred: false, + timestamp_inferred: false, + account_identity_source: IdentitySource::Unresolved, + }); + event +} + +fn classes(calls: &[AnalyzedCacheCall]) -> Vec { + calls.iter().map(|call| call.verdict.class).collect() +} + +fn losses(calls: &[AnalyzedCacheCall]) -> Vec> { + calls.iter().map(|call| call.verdict.loss).collect() +} + +fn health(calls: &[AnalyzedCacheCall]) -> CacheHealthV1 { + let mut health = CacheHealthV1::default(); + for call in calls { + health.add_call(call); + } + health +} + +#[test] +fn gap_bins_are_closed_below_and_open_above() { + let cases = [ + (0, 0), + (59, 0), + (60, 1), + (299, 1), + (300, 2), + (599, 2), + (600, 3), + (899, 3), + (900, 4), + (1_799, 4), + (1_800, 5), + (2_699, 5), + (2_700, 6), + (3_599, 6), + (3_600, 7), + (86_400, 7), + ]; + for (seconds, bin) in cases { + assert_eq!(cache_gap_bin(seconds), bin, "{seconds}s"); + } + let labels = (0..CACHE_GAP_BIN_COUNT) + .map(cache_gap_bin_label) + .collect::>(); + assert_eq!( + labels, + ["<1m", "1–5m", "5–10m", "10–15m", "15–30m", "30–45m", "45–60m", "60m+"] + ); +} + +#[test] +fn hit_ratio_counts_cache_writes_in_the_denominator() { + assert_eq!(cache_hit_ratio(100, 300, 600), Some(0.6)); + assert_eq!(cache_hit_ratio(0, 0, 0), None); +} + +#[test] +fn growing_context_is_reuse_not_loss() { + let events = [ + call(1, 0, 10, 0, Some(20_000)), + call(2, 30, 10, 20_000, Some(2_000)), + call(3, 60, 10, 22_000, Some(3_000)), + call(4, 90, 10, 25_000, Some(500)), + ]; + let calls = analyze_cache_calls(&events); + assert_eq!( + classes(&calls), + [ + CacheCallClass::FirstObserved, + CacheCallClass::Comparable, + CacheCallClass::Comparable, + CacheCallClass::Comparable + ] + ); + assert!(losses(&calls).iter().all(Option::is_none)); + assert_eq!(calls[2].verdict.baseline_tokens, Some(22_000)); + let health = health(&calls); + assert_eq!((health.first_observed, health.first_cold), (1, 1)); + assert_eq!(health.gap_comparable[0], 3); +} + +#[test] +fn losses_need_half_the_baseline_and_the_token_floor() { + // Each pair: a call leaving 10,000 reusable tokens, then the probe, which + // writes to the cache whatever it did not read. + let probe = |read: u64| { + let events = [ + call(1, 0, 0, 10_000, Some(0)), + call(2, 10, 0, read, Some(20_000 - read)), + ]; + analyze_cache_calls(&events)[1].verdict.loss + }; + assert_eq!(probe(0), Some(CacheLoss::Full)); + assert_eq!(probe(5_000), Some(CacheLoss::Partial)); + assert_eq!(probe(5_001), None, "a drop under half is not a loss"); + + // A drop of more than half that is still under the token floor. + let events = [ + call(1, 0, 0, 8_000, Some(0)), + call(2, 10, 8_000, 3_905, Some(0)), + ]; + assert_eq!(analyze_cache_calls(&events)[1].verdict.loss, None); + let events = [ + call(1, 0, 0, 8_192, Some(0)), + call(2, 10, 8_192, 4_096, Some(0)), + ]; + assert_eq!( + analyze_cache_calls(&events)[1].verdict.loss, + Some(CacheLoss::Partial) + ); +} + +#[test] +fn the_baseline_is_bounded_by_the_current_input() { + let events = [ + call(1, 0, 0, 100_000, Some(0)), + // The context shrank to 20,000 tokens; 15,000 of them were reused. + call(2, 10, 5_000, 15_000, Some(0)), + ]; + let verdict = &analyze_cache_calls(&events)[1].verdict; + assert_eq!(verdict.baseline_tokens, Some(20_000)); + assert_eq!(verdict.loss, None); +} + +#[test] +fn without_write_telemetry_previous_reads_are_the_baseline() { + let events = [ + call(1, 0, 40_000, 50_000, None), + call(2, 10, 45_000, 50_000, None), + ]; + let calls = analyze_cache_calls(&events); + assert_eq!(calls[1].verdict.baseline_tokens, Some(50_000)); + assert_eq!(calls[1].verdict.loss, None); +} + +#[test] +fn a_tiny_previous_context_is_too_small_to_judge() { + let events = [ + call(1, 0, 100, 0, Some(3_000)), + call(2, 10, 100, 0, Some(3_000)), + ]; + let calls = analyze_cache_calls(&events); + assert_eq!(calls[1].verdict.class, CacheCallClass::BelowThreshold); + assert_eq!(health(&calls).below_threshold, 1); +} + +#[test] +fn sub_agents_are_separate_streams() { + let mut events = vec![ + call(1, 0, 0, 100_000, Some(1_000)), + call(2, 20, 0, 0, Some(20_000)), + call(3, 40, 0, 101_000, Some(1_000)), + call(4, 60, 0, 20_000, Some(500)), + ]; + // In one stream, the small agent context reads as a drop. + assert!(analyze_cache_calls(&events) + .iter() + .any(|call| call.verdict.loss.is_some())); + + for index in [1, 3] { + events[index].context = Some(CallContext { + agent_hash: Some("agent".to_string()), + ..CallContext::default() + }); + } + let calls = analyze_cache_calls(&events); + assert!(losses(&calls).iter().all(Option::is_none)); + assert_eq!(health(&calls).first_observed, 2); + + // A different transcript is a different stream too. + let mut events = vec![ + call(1, 0, 0, 100_000, Some(1_000)), + call(2, 20, 0, 5_000, Some(500)), + ]; + if let Some(evidence) = events[1].parse_evidence.as_mut() { + evidence.source_file_path_hash = Some("agent-file".to_string()); + } + assert_eq!(health(&analyze_cache_calls(&events)).first_observed, 2); +} + +#[test] +fn model_changes_and_compaction_restart_the_comparison() { + let mut events = vec![ + call(1, 0, 0, 100_000, Some(0)), + call(2, 10, 0, 0, Some(100_000)), + call(3, 20, 0, 100_000, Some(0)), + call(4, 30, 0, 0, Some(30_000)), + ]; + events[1].model = Some(ModelInfo { + normalized_name: Some("other-model".to_string()), + ..ModelInfo::default() + }); + events[2].model = events[1].model.clone(); + events[3].model = events[1].model.clone(); + events[3].context = Some(CallContext { + after_compaction: true, + ..CallContext::default() + }); + let calls = analyze_cache_calls(&events); + assert_eq!( + calls[1].verdict.boundary, + Some(CacheBoundaryReason::ModelChange) + ); + assert_eq!( + calls[1].verdict.previous_model.as_deref(), + Some("claude-model") + ); + assert_eq!(calls[2].verdict.class, CacheCallClass::Comparable); + assert_eq!( + calls[3].verdict.boundary, + Some(CacheBoundaryReason::Compaction) + ); + assert!(losses(&calls).iter().all(Option::is_none)); + assert_eq!(health(&calls).boundaries, 2); +} + +#[test] +fn a_call_without_a_timestamp_is_counted_but_not_placed() { + let mut events = vec![ + call(1, 0, 0, 50_000, Some(0)), + call(2, 10, 0, 0, Some(0)), + call(3, 20, 0, 50_000, Some(0)), + ]; + if let Some(evidence) = events[1].parse_evidence.as_mut() { + evidence.timestamp_inferred = true; + } + let calls = analyze_cache_calls(&events); + let unplaced = calls + .iter() + .find(|call| call.event_id.0 == "event_002") + .expect("unplaced call"); + assert_eq!( + unplaced.verdict.unclassifiable, + Some(CacheUnclassifiableReason::TimestampMissing) + ); + let third = calls + .iter() + .find(|call| call.event_id.0 == "event_003") + .expect("third call"); + assert_eq!(third.verdict.class, CacheCallClass::Comparable); + assert_eq!(third.verdict.loss, None); +} + +#[test] +fn calls_with_one_timestamp_and_no_recorded_order_are_unclassifiable() { + // The cold call sorts first by event id, but either could have come first, + // so neither is judged against the call before them. + let mut events = vec![ + call(1, 0, 0, 50_000, Some(0)), + call(2, 10, 0, 0, Some(0)), + call(3, 10, 0, 50_000, Some(0)), + call(4, 20, 0, 50_000, Some(0)), + ]; + for event in &mut events[1..3] { + if let Some(evidence) = event.parse_evidence.as_mut() { + evidence.source_line_number = None; + } + } + let calls = analyze_cache_calls(&events); + for tied in &calls[1..3] { + assert_eq!( + tied.verdict.unclassifiable, + Some(CacheUnclassifiableReason::AmbiguousOrder) + ); + } + assert_eq!( + calls[3].verdict.boundary, + Some(CacheBoundaryReason::EvidenceGap) + ); + assert!(losses(&calls).iter().all(Option::is_none)); + + // The same timestamps keep their order when the lines say it. + let events = [ + call(1, 0, 0, 50_000, Some(0)), + call(2, 10, 0, 50_000, Some(0)), + call(3, 10, 0, 0, Some(50_000)), + ]; + assert_eq!( + analyze_cache_calls(&events)[2].verdict.loss, + Some(CacheLoss::Full) + ); +} + +#[test] +fn aggregated_usage_is_unclassifiable_and_breaks_the_chain() { + let mut events = vec![ + call(1, 0, 0, 80_000, Some(0)), + call(2, 60, 3_000, 240_000, Some(0)), + call(3, 120, 0, 0, Some(80_000)), + ]; + events[1].usage.requests = Some(3); + let calls = analyze_cache_calls(&events); + assert_eq!( + calls[1].verdict.unclassifiable, + Some(CacheUnclassifiableReason::Aggregated) + ); + assert_eq!(calls[1].requests, 3); + assert_eq!( + calls[2].verdict.boundary, + Some(CacheBoundaryReason::EvidenceGap) + ); + let health = health(&calls); + assert_eq!( + (health.requests, health.unclassifiable, health.analyzed()), + (5, 3, 2) + ); +} + +#[test] +fn recorded_calls_of_a_turn_are_judged_one_by_one() { + let mut turn = call(1, 0, 0, 0, Some(0)); + turn.provider = "codex".to_string(); + turn.usage.requests = Some(3); + turn.context = Some(CallContext { + calls: vec![ + ModelCall { + completed_at: at(10), + requested_at: Some(at(0)), + input_tokens: Some(30_000), + cache_read_tokens: Some(0), + cache_creation_tokens: Some(0), + after_compaction: false, + model: None, + }, + ModelCall { + completed_at: at(20), + requested_at: Some(at(12)), + input_tokens: Some(2_000), + cache_read_tokens: Some(29_952), + cache_creation_tokens: Some(0), + after_compaction: false, + model: None, + }, + ModelCall { + completed_at: at(400), + requested_at: Some(at(390)), + input_tokens: Some(31_000), + cache_read_tokens: Some(1_024), + cache_creation_tokens: Some(0), + after_compaction: false, + model: None, + }, + ], + ..CallContext::default() + }); + let calls = analyze_cache_calls([&turn]); + assert_eq!(calls.len(), 3); + assert_eq!(calls[1].verdict.loss, None); + assert_eq!(calls[2].verdict.loss, Some(CacheLoss::Partial)); + assert_eq!(calls[2].verdict.gap_seconds, Some(370)); + assert_eq!( + calls[2].verdict.gap_timing, + Some(CacheGapTiming::RequestStart) + ); + + // A breakdown that does not cover every call is not used. + if let Some(context) = turn.context.as_mut() { + context.calls.pop(); + } + let calls = analyze_cache_calls([&turn]); + assert_eq!( + calls[0].verdict.unclassifiable, + Some(CacheUnclassifiableReason::Aggregated) + ); +} + +#[test] +fn a_model_change_inside_a_turn_is_a_boundary_not_a_loss() { + let mut turn = call(1, 0, 0, 0, Some(0)); + turn.provider = "codex".to_string(); + turn.usage.requests = Some(2); + let model_call = |seconds, read, write, model: &str| ModelCall { + completed_at: at(seconds), + requested_at: None, + input_tokens: Some(1_000), + cache_read_tokens: Some(read), + cache_creation_tokens: Some(write), + after_compaction: false, + model: Some(model.to_string()), + }; + turn.context = Some(CallContext { + calls: vec![ + model_call(10, 50_000, 0, "gpt-5"), + model_call(20, 0, 0, "gpt-5.1"), + ], + ..CallContext::default() + }); + let calls = analyze_cache_calls([&turn]); + assert_eq!( + calls[1].verdict.boundary, + Some(CacheBoundaryReason::ModelChange) + ); + assert_eq!(calls[1].verdict.previous_model.as_deref(), Some("gpt-5")); + assert_eq!(calls[1].verdict.loss, None); +} + +#[test] +fn a_turn_call_without_input_counts_is_unclassifiable() { + let mut turn = call(1, 0, 0, 0, Some(0)); + turn.provider = "codex".to_string(); + turn.usage.requests = Some(3); + let model_call = |seconds, input: Option, read: Option| ModelCall { + completed_at: at(seconds), + requested_at: None, + input_tokens: input, + cache_read_tokens: read, + cache_creation_tokens: None, + after_compaction: false, + model: None, + }; + turn.context = Some(CallContext { + calls: vec![ + model_call(10, Some(1_000), Some(50_000)), + model_call(20, None, None), + model_call(30, Some(1_000), Some(0)), + ], + ..CallContext::default() + }); + let calls = analyze_cache_calls([&turn]); + assert_eq!( + calls[1].verdict.unclassifiable, + Some(CacheUnclassifiableReason::NoInputTelemetry) + ); + // The call after it has no known predecessor, so it is not a loss. + assert_eq!( + calls[2].verdict.boundary, + Some(CacheBoundaryReason::EvidenceGap) + ); + assert_eq!(calls[2].verdict.loss, None); +} + +#[test] +fn gaps_prefer_the_recorded_request_start() { + let mut events = vec![ + call(1, 0, 0, 50_000, Some(0)), + call(2, 400, 0, 50_000, Some(0)), + call(3, 800, 0, 50_000, Some(0)), + ]; + events[1].context = Some(CallContext { + requested_at: Some(at(330)), + ..CallContext::default() + }); + // A request start before the previous response belongs to another exchange. + events[2].context = Some(CallContext { + requested_at: Some(at(390)), + ..CallContext::default() + }); + let calls = analyze_cache_calls(&events); + assert_eq!(calls[1].verdict.gap_seconds, Some(330)); + assert_eq!( + calls[1].verdict.gap_timing, + Some(CacheGapTiming::RequestStart) + ); + assert_eq!(calls[2].verdict.gap_seconds, Some(400)); + assert_eq!( + calls[2].verdict.gap_timing, + Some(CacheGapTiming::RecordedEvents) + ); + let health = health(&calls); + assert_eq!(health.timing_estimated, 1); + assert_eq!(health.gap_comparable[2], 2); +} + +#[test] +fn a_short_gap_loss_is_counted_in_its_bin() { + let events = [ + call(1, 0, 2, 350_000, Some(1_000)), + call(2, 40, 2, 27_000, Some(325_000)), + call(3, 80, 2, 352_000, Some(2_000)), + ]; + let calls = analyze_cache_calls(&events); + assert_eq!(losses(&calls), [None, Some(CacheLoss::Partial), None]); + let health = health(&calls); + assert_eq!((health.gap_comparable[0], health.gap_losses[0]), (2, 1)); + assert_eq!((health.losses, health.full_losses), (1, 0)); +} + +#[test] +fn a_loss_after_a_long_pause_is_counted_in_the_open_bin() { + let events = [ + call(1, 0, 2, 120_000, Some(1_000)), + call(2, 2 * 3_600, 2, 0, Some(121_500)), + ]; + let calls = analyze_cache_calls(&events); + assert_eq!(calls[1].verdict.loss, Some(CacheLoss::Full)); + let health = health(&calls); + assert_eq!((health.gap_comparable[7], health.gap_losses[7]), (1, 1)); +} + +#[test] +fn previous_write_lifetimes_are_kept_as_evidence() { + let mut events = vec![ + call(1, 0, 2, 10_000, Some(9_000)), + call(2, 600, 2, 0, Some(19_000)), + ]; + events[0].usage.cache_creation_5m_tokens = Some(4_000); + events[0].usage.cache_creation_1h_tokens = Some(5_000); + let calls = analyze_cache_calls(&events); + assert_eq!(calls[0].write_ttl, Some(CacheWriteTtl::Mixed)); + assert_eq!( + calls[1].verdict.previous_write_ttl, + Some(CacheWriteTtl::Mixed) + ); +} + +#[test] +fn health_counts_add_up_and_merge() { + let mut events = vec![ + call(1, 0, 0, 0, Some(40_000)), + call(2, 30, 0, 40_000, Some(1_000)), + call(3, 400, 41_000, 0, Some(0)), + call(4, 460, 0, 41_000, Some(0)), + call(5, 520, 9_000, 0, Some(0)), + ]; + events[4].usage.requests = Some(2); + let calls = analyze_cache_calls(&events); + let health = health(&calls); + assert_eq!(health.uncached, 1, "the third call skipped the cache"); + assert_eq!( + health.requests, + health.unclassifiable + + health.first_observed + + health.boundaries + + health.below_threshold + + health.uncached + + health.comparable + ); + assert_eq!(health.gap_comparable.iter().sum::(), health.comparable); + assert_eq!(health.gap_losses.iter().sum::(), health.losses); + assert_eq!( + health.gap_missed_tokens.iter().sum::(), + health.missed_tokens + ); + + let mut merged = CacheHealthV1::default(); + merged.merge(&health); + merged.merge(&health); + assert_eq!(merged.requests, health.requests * 2); + assert_eq!(merged.gap_comparable[2], health.gap_comparable[2] * 2); +} + +#[test] +fn providers_without_call_order_are_left_out() { + let mut event = call(1, 0, 0, 1_000, Some(0)); + event.provider = "cursor".to_string(); + assert!(analyze_cache_calls([&event]).is_empty()); +} + +#[test] +fn the_analysis_does_not_depend_on_input_order() { + let events = vec![ + call(1, 0, 0, 0, Some(30_000)), + call(2, 20, 0, 30_000, Some(0)), + call(3, 700, 30_000, 0, Some(0)), + ]; + let mut reversed = events.clone(); + reversed.reverse(); + assert_eq!(analyze_cache_calls(&events), analyze_cache_calls(&reversed)); +} + +#[test] +fn the_report_splits_totals_by_provider_and_utc_day() { + let mut events = vec![ + call(1, 13 * 3_600 + 59 * 60, 100, 0, Some(20_000)), + call(2, 14 * 3_600 + 1, 100, 20_000, Some(100)), + ]; + let mut cursor = call(3, 0, 1_000, 3_000, None); + cursor.provider = "cursor".to_string(); + events.push(cursor); + let calls = analyze_cache_calls(&events); + let report = build_cache_report( + CacheReportInput { + label: "fixture".to_string(), + since: None, + until: at(86_400), + filters: CacheReportFilters::default(), + events: &events, + summaries: &[], + calls, + include_details: false, + }, + Some(&FixedOffset::east_opt(5 * 3_600 + 30 * 60).unwrap()), + ); + assert_eq!(report.totals.requests, 3); + assert_eq!(report.totals.logical_input_tokens, 44_300); + assert_eq!(report.providers.len(), 2); + assert!(report.providers[1].diagnostics.is_none(), "cursor"); + assert_eq!( + report.diagnostics.write_split_unavailable_input_tokens, 0, + "only covered providers count unsplit input" + ); + assert_eq!( + report + .days + .iter() + .map(|day| day.day.as_str()) + .collect::>(), + ["2026-03-02", "2026-03-03"] + ); + let timeline = report.timeline.expect("timeline"); + assert_eq!(timeline.len(), 2); + assert_eq!(timeline[0].start, "2026-03-03T05:20:00+05:30"); + assert_eq!(report.gap_histogram[0].comparable, 0); + assert_eq!(report.gap_histogram[1].label, "1–5m"); + assert!(report.details.is_none()); +} + +#[test] +fn call_context_is_omitted_when_empty_and_optional_on_read() { + let event = call(1, 0, 10, 20, Some(30)); + let value = serde_json::to_value(&event).expect("event json"); + assert!(value.get("context").is_none()); + let mut legacy = value.clone(); + legacy.as_object_mut().expect("object").remove("context"); + let read: UsageEvent = serde_json::from_value(legacy).expect("legacy event"); + assert_eq!(read.context, None); + + let context = CallContext { + requested_at: Some(at(0)), + ..CallContext::default() + }; + assert_eq!( + serde_json::to_value(&context).expect("context json"), + serde_json::json!({ "requested_at": "2026-03-02T10:00:00Z" }) + ); +} + +#[test] +fn removing_cache_health_restores_the_summary_a_receiver_stores() { + let metrics = |cache_health| SummaryMetrics { + active_seconds: None, + tracked_requests: None, + tracked_output_tokens: None, + tracked_reasoning_tokens: None, + latency_ms: None, + time_to_first_token_ms: None, + generated_tps: None, + visible_tps: None, + overall_generated_tps: None, + overall_visible_tps: None, + cache_hit_ratio: None, + reasoning_share: None, + total_messages: Some(4), + user_messages: None, + assistant_messages: None, + developer_messages: None, + cache_health, + }; + let only_health = metrics(Some(CacheHealthV1::default())); + let mut kept = only_health.clone(); + kept.total_messages = Some(4); + let mut empty = only_health; + empty.total_messages = None; + assert!(!empty.is_empty()); + + let summary = |metrics| UsageSummary { + metrics, + ..serde_json::from_value::(fixture_summary_json()).expect("summary") + }; + assert_eq!( + summary(Some(empty)).without_cache_health().metrics, + None, + "metrics holding only cache_health go away" + ); + assert_eq!( + summary(Some(kept)).without_cache_health().metrics, + Some(metrics(None)) + ); +} + +fn fixture_summary_json() -> serde_json::Value { + serde_json::json!({ + "schema_version": USAGE_SUMMARY_SCHEMA_VERSION, + "summary_id": "summary_fixture", + "device_id": "device", + "provider": "claude_code", + "source_id": "source", + "provider_account_id": null, + "source": { + "adapter_id": "test", + "adapter_version": "0", + "source_kind": "local_adapter", + "location_origin": null, + "source_type": "jsonl", + "source_path_hash": null, + "source_record_id": null, + "parse_confidence": "high" + }, + "model": null, + "usage": {}, + "cost": { "currency": "USD", "estimated_api_equivalent_usd": null, "provider_reported_usd": null, "pricing_source": null, "pricing_version": null, "confidence": "low" }, + "parse_evidence": null, + "project": null, + "privacy": { "mode": "metadata_only", "contains_prompt_text": false, "contains_response_text": false, "contains_file_paths": false }, + "metrics": null, + "period_start": null, + "period_end": null, + "observed_at": "2026-03-02T10:00:00Z", + "metadata": { "summary_format": "daily_rollup.v1", "summary_version": null, "total_sessions": null, "total_messages": null, "last_computed_at": null }, + "imported_at": "2026-03-02T10:00:00Z" + }) +} + +#[test] +fn a_call_that_neither_reads_nor_writes_the_cache_is_uncached_not_a_loss() { + // A router serving the request without the cache, between cached calls. + let events = [ + call(1, 0, 10, 0, Some(50_000)), + call(2, 10, 50_010, 0, Some(0)), + call(3, 20, 51_000, 0, Some(0)), + call(4, 30, 10, 50_000, Some(1_000)), + ]; + let calls = analyze_cache_calls(&events); + assert_eq!( + calls + .iter() + .map(|call| call.verdict.class) + .collect::>(), + [ + CacheCallClass::FirstObserved, + CacheCallClass::Uncached, + CacheCallClass::Uncached, + CacheCallClass::BelowThreshold, + ] + ); + assert!(losses(&calls).iter().all(Option::is_none)); + let health = health(&calls); + assert_eq!((health.uncached, health.losses), (2, 0)); + + // Codex reports zero writes on calls that use the cache, so a cold Codex + // call is still judged. + let mut codex = events.clone(); + for event in &mut codex { + event.provider = "codex".to_string(); + } + assert_eq!( + analyze_cache_calls(&codex)[1].verdict.loss, + Some(CacheLoss::Full) + ); +} + +#[test] +fn a_loss_records_the_reusable_context_it_missed() { + let events = [ + call(1, 0, 10, 0, Some(80_000)), + call(2, 30, 10, 80_000, Some(0)), + call(3, 3_900, 10, 20_000, Some(60_000)), + ]; + let calls = analyze_cache_calls(&events); + assert_eq!(calls[1].verdict.missed_tokens, None); + assert_eq!(calls[2].verdict.loss, Some(CacheLoss::Partial)); + assert_eq!(calls[2].verdict.missed_tokens, Some(60_000)); + + let health = health(&calls); + assert_eq!(health.missed_tokens, 60_000); + assert_eq!(health.gap_missed_tokens[CACHE_GAP_BIN_COUNT - 1], 60_000); + let mut doubled = health.clone(); + doubled.merge(&health); + assert_eq!(doubled.missed_tokens, 120_000); + assert_eq!( + gap_histogram(&doubled)[CACHE_GAP_BIN_COUNT - 1].missed_tokens, + 120_000 + ); +} + +#[test] +fn an_inferred_model_does_not_make_a_model_change() { + let mut events = vec![ + call(1, 0, 0, 30_000, Some(0)), + call(2, 10, 0, 1_000, Some(29_000)), + ]; + // The first model is the parser's fallback, the second one the transcript named. + if let Some(evidence) = events[0].parse_evidence.as_mut() { + evidence.model_inferred = true; + } + events[1].model = Some(ModelInfo { + normalized_name: Some("other-model".to_string()), + ..ModelInfo::default() + }); + let calls = analyze_cache_calls(&events); + assert_eq!(calls[1].verdict.boundary, None); + assert_eq!(calls[1].verdict.loss, Some(CacheLoss::Partial)); + assert_eq!(calls[1].verdict.missed_tokens, Some(29_000)); +} + +#[test] +fn unsplit_input_is_counted_per_call_when_writes_are_partly_reported() { + let mut turn = call(1, 0, 3_000, 0, Some(0)); + turn.provider = "codex".to_string(); + turn.usage.requests = Some(2); + let model_call = |seconds, input, write| ModelCall { + completed_at: at(seconds), + requested_at: None, + input_tokens: Some(input), + cache_read_tokens: Some(0), + cache_creation_tokens: write, + after_compaction: false, + model: None, + }; + // The turn's summed usage says zero writes; one call never reported them. + turn.context = Some(CallContext { + calls: vec![model_call(10, 1_000, Some(0)), model_call(20, 2_000, None)], + ..CallContext::default() + }); + let mut health = CacheHealthV1::default(); + health.add_event_input(&turn); + assert_eq!(health.write_split_unavailable_input_tokens, 2_000); + + // Without a call list the event's own counts decide. + let mut single = call(2, 30, 4_000, 0, None); + single.provider = "codex".to_string(); + health.add_event_input(&single); + assert_eq!(health.write_split_unavailable_input_tokens, 6_000); +} diff --git a/crates/statsai-core/src/tests/mod.rs b/crates/statsai-core/src/tests/mod.rs index b39a837..aa2817b 100644 --- a/crates/statsai-core/src/tests/mod.rs +++ b/crates/statsai-core/src/tests/mod.rs @@ -1,5 +1,6 @@ pub(crate) use super::*; +mod cache; mod report; mod support; mod types; diff --git a/crates/statsai-core/src/tests/support.rs b/crates/statsai-core/src/tests/support.rs index ce65aee..b311389 100644 --- a/crates/statsai-core/src/tests/support.rs +++ b/crates/statsai-core/src/tests/support.rs @@ -74,6 +74,7 @@ pub(super) fn test_event( }, created_at: started_at, imported_at: started_at, + context: None, } } diff --git a/crates/statsai-core/src/types/event.rs b/crates/statsai-core/src/types/event.rs index acc7695..76d1bf0 100644 --- a/crates/statsai-core/src/types/event.rs +++ b/crates/statsai-core/src/types/event.rs @@ -396,6 +396,54 @@ pub struct PrivacyInfo { pub contains_file_paths: bool, } +/// Where a usage event sits in its conversation, for the prompt-cache report. +/// +/// Recorded only by adapters that can see the boundaries, and kept on the +/// device: sync strips it from events and only bounded counts leave in daily +/// summaries. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize, JsonSchema)] +pub struct CallContext { + /// Hashed sub-agent identity. Sub-agents share their parent's session but + /// keep their own context, so each one is a separate cache stream. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub agent_hash: Option, + /// When the request was sent, read from the record that triggered it: a + /// prompt or a tool result. Single-call events only. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub requested_at: Option>, + /// The provider compacted the conversation before this request. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub after_compaction: bool, + /// The model calls inside an event that aggregates several, such as a Codex + /// turn, oldest first. Empty for single-call events. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub calls: Vec, +} + +/// One model call inside an aggregated usage event. Token fields follow the +/// additive `UsageCounts` contract: ordinary input excludes cache reads and +/// writes, and `None` means the provider did not report the count. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct ModelCall { + pub completed_at: DateTime, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub requested_at: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_tokens: Option, + /// `None` when the provider did not report cache writes for this call. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_tokens: Option, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub after_compaction: bool, + /// The model that served this call, as `cache_model_name` names it. + /// `None` when the transcript did not say, so a model change cannot be + /// read from it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model: Option, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct UsageEvent { pub schema_version: String, @@ -417,6 +465,10 @@ pub struct UsageEvent { pub privacy: PrivacyInfo, pub created_at: DateTime, pub imported_at: DateTime, + /// Local only: sync removes it, so it is not part of the sync schema. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(skip)] + pub context: Option, } impl UsageCounts { diff --git a/crates/statsai-core/src/types/summary.rs b/crates/statsai-core/src/types/summary.rs index f7c1f71..96171e7 100644 --- a/crates/statsai-core/src/types/summary.rs +++ b/crates/statsai-core/src/types/summary.rs @@ -2,6 +2,7 @@ use super::event::{ CostInfo, EventSource, MetricStats, ModelInfo, ParseEvidence, PrivacyInfo, ProjectInfo, UsageCounts, }; +use crate::cache::CacheHealthV1; use crate::ids::{ProviderAccountId, SourceId, SummaryId}; use chrono::{DateTime, Utc}; use schemars::JsonSchema; @@ -30,6 +31,48 @@ pub struct SummaryMetrics { pub user_messages: Option, pub assistant_messages: Option, pub developer_messages: Option, + /// Prompt-cache diagnostics, for providers whose events record call order. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_health: Option, +} + +impl SummaryMetrics { + /// Whether no metric is set, in which case a summary carries `None`. + #[must_use] + pub fn is_empty(&self) -> bool { + self.active_seconds.is_none() + && self.tracked_requests.is_none() + && self.tracked_output_tokens.is_none() + && self.tracked_reasoning_tokens.is_none() + && self.latency_ms.is_none() + && self.time_to_first_token_ms.is_none() + && self.generated_tps.is_none() + && self.visible_tps.is_none() + && self.overall_generated_tps.is_none() + && self.overall_visible_tps.is_none() + && self.cache_hit_ratio.is_none() + && self.reasoning_share.is_none() + && self.total_messages.is_none() + && self.user_messages.is_none() + && self.assistant_messages.is_none() + && self.developer_messages.is_none() + && self.cache_health.is_none() + } +} + +impl UsageSummary { + /// The summary as a receiver without `cache_health` support stores it: + /// the object removed, and metrics dropped when nothing else is left. + #[must_use] + pub fn without_cache_health(mut self) -> Self { + if let Some(metrics) = self.metrics.as_mut() { + metrics.cache_health = None; + } + if self.metrics.as_ref().is_some_and(SummaryMetrics::is_empty) { + self.metrics = None; + } + self + } } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] diff --git a/crates/statsai-daemon/Cargo.toml b/crates/statsai-daemon/Cargo.toml index a510eb2..7b700c6 100644 --- a/crates/statsai-daemon/Cargo.toml +++ b/crates/statsai-daemon/Cargo.toml @@ -21,6 +21,7 @@ chunked_transfer = "1" httpdate = "1" log = "0.4" notify = { workspace = true, optional = true } +percent-encoding.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/crates/statsai-daemon/src/lib.rs b/crates/statsai-daemon/src/lib.rs index 4a59f33..f744f8c 100644 --- a/crates/statsai-daemon/src/lib.rs +++ b/crates/statsai-daemon/src/lib.rs @@ -89,6 +89,16 @@ fn handle_request(mut request: Request, store: &Arc>, auth_token: & return respond_json(request, StatusCode(200), &health_payload()); } + let (path, query) = url.split_once('?').unwrap_or((url.as_str(), "")); + if path == "/reports/cache" { + let report_query = match cache_report_query(query, chrono::Utc::now()) { + Ok(report_query) => report_query, + Err(error) => return respond_text(request, StatusCode(400), &format!("{error:#}")), + }; + let report = lock_store(store).cache_report(&report_query, &chrono::Local)?; + return respond_json(request, StatusCode(200), &report); + } + let s = lock_store(store); let payload = match url.as_str() { "/status" => json!({ @@ -115,6 +125,47 @@ fn handle_request(mut request: Request, store: &Arc>, auth_token: & respond_json(request, StatusCode(200), &payload) } +/// Reads `from`, `to`, `provider`, `account`, `session`, and `timeline` from +/// a percent-encoded query. Per-call details are left to the CLI. +fn cache_report_query( + query: &str, + now: chrono::DateTime, +) -> Result { + let mut params = std::collections::BTreeMap::new(); + for pair in query.split('&').filter(|pair| !pair.is_empty()) { + let (key, value) = pair.split_once('=').unwrap_or((pair, "")); + let value = percent_encoding::percent_decode_str(value) + .decode_utf8() + .context("query values must be UTF-8")?; + if params.insert(key, value.into_owned()).is_some() { + bail!("repeated query parameter {key}"); + } + } + let (from, to) = (params.remove("from"), params.remove("to")); + let mut report_query = match params.remove("all").as_deref() { + None | Some("0" | "false") => { + statsai_store::CacheReportQuery::for_range(from.as_deref(), to.as_deref(), now)? + } + Some("1" | "true") if from.is_none() && to.is_none() => { + statsai_store::CacheReportQuery::all_time(now) + } + Some("1" | "true") => bail!("all cannot be combined with from or to"), + Some(other) => bail!("all must be true or false, not {other}"), + }; + report_query.provider = params.remove("provider"); + report_query.account = params.remove("account"); + report_query.session = params.remove("session"); + report_query.timeline = match params.remove("timeline").as_deref() { + None | Some("0" | "false") => false, + Some("1" | "true") => true, + Some(other) => bail!("timeline must be true or false, not {other}"), + }; + if let Some(key) = params.keys().next() { + bail!("unknown query parameter {key}"); + } + Ok(report_query) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct HttpRejection { status: StatusCode, diff --git a/crates/statsai-daemon/src/tests.rs b/crates/statsai-daemon/src/tests.rs index 040055a..dc82459 100644 --- a/crates/statsai-daemon/src/tests.rs +++ b/crates/statsai-daemon/src/tests.rs @@ -910,3 +910,127 @@ fn test_task_verification() -> TaskVerification { updated_at: created_at, } } + +#[test] +fn cache_report_query_reads_percent_encoded_filters() { + let now = Utc.with_ymd_and_hms(2026, 7, 10, 12, 0, 0).unwrap(); + let query = cache_report_query( + "from=2026-07-01&to=2026-07-03T00%3A00%3A00%2B02%3A00&provider=claude_code&account=unassigned&session=session_ab&timeline=true", + now, + ) + .expect("query"); + assert_eq!( + query.since, + Some(Utc.with_ymd_and_hms(2026, 7, 1, 0, 0, 0).unwrap()) + ); + assert_eq!( + query.until, + Utc.with_ymd_and_hms(2026, 7, 2, 22, 0, 0).unwrap() + ); + assert_eq!(query.provider.as_deref(), Some("claude_code")); + assert_eq!(query.account.as_deref(), Some("unassigned")); + assert_eq!(query.session.as_deref(), Some("session_ab")); + assert!(query.timeline); + assert!(!query.details, "per-call details stay with the CLI"); + + let default = cache_report_query("", now).expect("default"); + assert_eq!(default.since, Some(now - chrono::Duration::days(7))); + let all = cache_report_query("all=true", now).expect("all"); + assert_eq!((all.since, all.until), (None, now)); + for rejected in [ + "details=true", + "from=a&from=b", + "timeline=maybe", + "from=not-a-date", + "all=true&from=2026-07-01", + "all=maybe", + ] { + assert!(cache_report_query(rejected, now).is_err(), "{rejected}"); + } +} + +#[test] +fn cache_report_route_requires_the_bearer_token() { + assert_eq!( + validate_http_request( + &Method::Get, + "/reports/cache?from=2026-07-01", + &[], + None, + "secret" + ), + Err(HttpRejection { + status: StatusCode(401), + message: "missing or invalid bearer token", + }) + ); +} + +#[test] +fn the_cache_report_route_serves_the_cli_report() { + let store = Store::in_memory().expect("store"); + let event = |id: &str, at: &str, read: u64, write: u64| -> statsai_core::UsageEvent { + serde_json::from_value(json!({ + "schema_version": "usage_event.v1", + "event_id": id, + "device_id": "device", + "provider": "claude_code", + "source_id": "src_daemon_cache", + "provider_account_id": null, + "subscription_id": null, + "source": { + "adapter_id": "test", "adapter_version": "0", "source_kind": "local_adapter", + "location_origin": null, "source_type": "jsonl", "source_path_hash": null, + "source_record_id": null, "parse_confidence": "high" + }, + "session": { + "session_id": "session_daemon", "local_session_id_hash": null, "title": null, + "started_at": at, "ended_at": null, "duration_seconds": null + }, + "model": null, + "usage": { "input_tokens": 10, "cache_read_tokens": read, "cache_creation_tokens": write, "requests": 1 }, + "runtime": null, + "cost": { "currency": "USD", "estimated_api_equivalent_usd": null, "provider_reported_usd": null, "pricing_source": null, "pricing_version": null, "confidence": "low" }, + "parse_evidence": null, + "project": null, + "git": null, + "privacy": { "mode": "metadata_only", "contains_prompt_text": false, "contains_response_text": false, "contains_file_paths": false }, + "created_at": at, + "imported_at": at + })) + .expect("event") + }; + store + .insert_events(&[ + event("event_a", "2026-07-01T10:00:00Z", 0, 50_000), + event("event_b", "2026-07-01T11:30:00Z", 0, 50_010), + ]) + .expect("insert"); + let expected = { + let query = cache_report_query("from=2026-07-01&to=2026-07-02", Utc::now()).expect("query"); + serde_json::to_value(store.cache_report(&query, &chrono::Local).expect("report")) + .expect("json") + }; + assert_eq!(expected["diagnostics"]["losses"], 1); + + let server = Server::http("127.0.0.1:0").expect("server"); + let address = server.server_addr().to_ip().expect("ip address"); + let store = Arc::new(Mutex::new(store)); + let handle = std::thread::spawn(move || { + let request = server.recv().expect("request"); + handle_request(request, &store, "secret").expect("handled"); + }); + let mut stream = std::net::TcpStream::connect(address).expect("connect"); + std::io::Write::write_all( + &mut stream, + b"GET /reports/cache?from=2026-07-01&to=2026-07-02 HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer secret\r\nConnection: close\r\n\r\n", + ) + .expect("request"); + let mut response = String::new(); + std::io::Read::read_to_string(&mut stream, &mut response).expect("response"); + handle.join().expect("server thread"); + assert!(response.starts_with("HTTP/1.1 200"), "{response}"); + let body = response.split_once("\r\n\r\n").expect("body").1; + let served: serde_json::Value = serde_json::from_str(body).expect("json body"); + assert_eq!(served, expected); +} diff --git a/crates/statsai-sdk/src/lib.rs b/crates/statsai-sdk/src/lib.rs index dd6ad9a..56a704b 100644 --- a/crates/statsai-sdk/src/lib.rs +++ b/crates/statsai-sdk/src/lib.rs @@ -8,6 +8,11 @@ pub use statsai_core as core; pub use statsai_store as store; pub use statsai_sync as sync; +/// The prompt-cache report: `Store::cache_report` with a [`CacheReportQuery`] +/// returns the same [`CacheReport`] that `statsai report cache --json` prints. +pub use statsai_core::{AnalyzedCacheCall, CacheHealthV1, CacheReport}; +pub use statsai_store::CacheReportQuery; + use anyhow::{bail, Result}; use chrono::{DateTime, Utc}; use schemars::JsonSchema; diff --git a/crates/statsai-store/src/account_plan/observations.rs b/crates/statsai-store/src/account_plan/observations.rs index 10dd3ea..21ff675 100644 --- a/crates/statsai-store/src/account_plan/observations.rs +++ b/crates/statsai-store/src/account_plan/observations.rs @@ -579,8 +579,7 @@ impl Store { dirty.extend(self.update_event_payload(event)?); changed += 1; } - self.refresh_sync_rollups_for_keys(&dirty.buckets)?; - self.refresh_session_rollups_for_keys(&dirty.sessions)?; + self.refresh_event_rollups(&dirty)?; Ok(changed) } diff --git a/crates/statsai-store/src/cache.rs b/crates/statsai-store/src/cache.rs new file mode 100644 index 0000000..920409a --- /dev/null +++ b/crates/statsai-store/src/cache.rs @@ -0,0 +1,614 @@ +//! Prompt-cache analysis over stored usage events. +//! +//! The detector compares each call with the one before it in its stream, so +//! it always reads whole sessions: a call on the first day of a report, or the +//! first call in a daily bucket, is judged against its real predecessor even +//! when that predecessor falls outside the range. A resumed session can follow +//! its predecessor by days, so no fixed look-back would do. Sessions are read +//! through `usage_events_session_started_idx`, which also bounds the lookup of +//! the calls after a changed one. + +use super::*; +use statsai_core::{ + analyze_cache_calls, build_cache_report, provider_has_cache_diagnostics, AnalyzedCacheCall, + CacheHealthV1, CacheReport, CacheReportFilters, CacheReportInput, CACHE_DIAGNOSTIC_PROVIDERS, + CACHE_HEALTH_VERSION, +}; +use std::collections::VecDeque; + +/// Records which detector version the stored daily summaries were built with. +const CACHE_HEALTH_ROLLUPS_METADATA_KEY: &str = "sync_rollups.cache_health_version"; + +/// Per-target record of whether an HTTP receiver accepts `cache_health`. +const SYNC_CACHE_HEALTH_METADATA_PREFIX: &str = "sync.cache_health_accepted:"; + +/// Sessions kept analyzed during one rollup refresh. Buckets refresh in day +/// order, so the sessions a bucket shares with the next one are the recent +/// ones; a bound keeps a full rebuild from holding every session at once. +const CACHE_SESSION_MEMO_CAPACITY: usize = 64; + +/// Account filter value for events no provider account was attributed to. +pub const CACHE_REPORT_UNASSIGNED_ACCOUNT: &str = "unassigned"; + +/// What `statsai report cache` and the local API ask for. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CacheReportQuery { + pub label: String, + pub since: Option>, + pub until: DateTime, + pub provider: Option, + /// A provider account id, or [`CACHE_REPORT_UNASSIGNED_ACCOUNT`]. + pub account: Option, + /// A hashed `session_…` id, or a prefix of one. + pub session: Option, + pub timeline: bool, + pub details: bool, +} + +impl CacheReportQuery { + /// Resolves `--from` and `--to` like the other reports. Date-only values + /// are UTC days, matching daily summaries; no bounds means the last seven + /// days. + pub fn for_range(from: Option<&str>, to: Option<&str>, now: DateTime) -> Result { + let period = match (from, to) { + (None, None) => statsai_core::ReportPeriod::LastDays(7), + (from, to) => statsai_core::report_period_from_range(from, to, now)?, + }; + Ok(Self::for_period(&period, now)) + } + + /// Every stored call, through now. + pub fn all_time(now: DateTime) -> Self { + Self::for_period(&statsai_core::ReportPeriod::AllTime, now) + } + + fn for_period(period: &statsai_core::ReportPeriod, now: DateTime) -> Self { + let (since, until) = period.published_window(now); + Self { + label: period.label(now), + since, + until, + provider: None, + account: None, + session: None, + timeline: false, + details: false, + } + } +} + +/// A session's calls, by the event that holds them. +type AnalyzedSession = HashMap>; + +/// The columns that place an event in its daily bucket, read without decoding +/// the payload. [`cache_bucket_key`] turns a row of them into the key. +const CACHE_BUCKET_COLUMNS_SQL: &str = "provider, source_id, provider_account_id, started_at, \ + CASE WHEN json_valid(payload) THEN json_extract(payload, '$.project') END"; + +/// Sessions analyzed during one rollup refresh, so the buckets a session spans +/// do not each re-read and re-classify it. +#[derive(Default)] +pub(crate) struct CacheSessionMemo { + sessions: HashMap, + order: VecDeque, +} + +impl Store { + pub fn cache_report( + &self, + query: &CacheReportQuery, + timeline_zone: &Tz, + ) -> Result + where + Tz::Offset: std::fmt::Display, + { + let mut events = Vec::new(); + let mut calls = Vec::new(); + for session_id in self.cache_report_sessions(query)? { + let mut session_events = Vec::new(); + let mut in_scope = BTreeSet::new(); + for (event, in_period) in self.cache_session_events_in_period(&session_id, query)? { + // Another provider's events can share the session id; they + // are counted with the providers that have no diagnostics. + if !provider_has_cache_diagnostics(&event.provider) { + continue; + } + if in_period && cache_report_includes(query, &event) { + in_scope.insert(event.event_id.0.clone()); + events.push(event.clone()); + } + session_events.push(event); + } + calls.extend( + analyze_and_price(&session_events) + .into_iter() + .filter(|call| in_scope.contains(&call.event_id.0)), + ); + } + // Other providers have no diagnostics but count toward the token totals. + events.extend( + self.cache_report_other_events(query)? + .into_iter() + .filter(|event| cache_report_includes(query, event)), + ); + events.sort_by(|left, right| { + left.session + .started_at + .cmp(&right.session.started_at) + .then_with(|| left.event_id.0.cmp(&right.event_id.0)) + }); + // Summaries with no events behind them sync alongside the daily + // rollups, so they count toward the same token totals. A session + // filter has no summaries to match. + let summaries = if query.session.is_some() { + Vec::new() + } else { + self.summaries()? + .into_iter() + .filter(|summary| { + is_http_rollup_passthrough_summary(summary) + && cache_report_includes_summary(query, summary) + }) + .collect::>() + }; + calls.sort_by(|left, right| { + left.completed_at + .cmp(&right.completed_at) + .then_with(|| left.event_id.0.cmp(&right.event_id.0)) + .then_with(|| left.call_index.cmp(&right.call_index)) + }); + Ok(build_cache_report( + CacheReportInput { + label: query.label.clone(), + since: query.since, + until: query.until, + filters: CacheReportFilters { + provider: query.provider.clone(), + account: query.account.clone(), + session: query.session.clone(), + }, + events: &events, + summaries: &summaries, + calls, + include_details: query.details, + }, + query.timeline.then_some(timeline_zone), + )) + } + + /// Records what an HTTP receiver's preflight said about `cache_health`. + pub fn record_sync_target_cache_health_support( + &self, + sink: &str, + target: &str, + accepted: bool, + ) -> Result<()> { + self.set_metadata_value( + &format!("{SYNC_CACHE_HEALTH_METADATA_PREFIX}{sink}:{target}"), + if accepted { "1" } else { "0" }, + ) + } + + /// Whether summaries for this target keep `cache_health`. An HTTP receiver + /// keeps it only after its preflight advertised support: one that would + /// discard the object must not be sent it, or the summary would look + /// synced with the diagnostics silently lost. Local sinks write what they + /// are given. + pub fn sync_target_accepts_cache_health(&self, sink: &str, target: &str) -> Result { + if sink != "http" { + return Ok(true); + } + Ok(self + .metadata_value(&format!( + "{SYNC_CACHE_HEALTH_METADATA_PREFIX}{sink}:{target}" + ))? + .as_deref() + == Some("1")) + } + + /// Diagnostics for one daily bucket, or `None` for providers the detector + /// does not cover. + pub(crate) fn bucket_cache_health( + &self, + key: &SyncRollupBucketKey, + events: &[UsageEvent], + memo: &mut CacheSessionMemo, + ) -> Result> { + if !provider_has_cache_diagnostics(&key.provider) { + return Ok(None); + } + let mut health = CacheHealthV1::default(); + for event in events { + health.add_event_input(event); + let session = self.analyzed_session(&event.session.session_id, memo)?; + for call in session.get(&event.event_id.0).into_iter().flatten() { + health.add_call(call); + } + } + Ok(Some(health)) + } + + /// Buckets holding calls that may follow a changed event in its stream. + /// + /// A call's verdict depends on its predecessor, so a late arrival, a + /// correction, or a deletion can change the bucket of the next call, which + /// may be a later day. Every event of the session at or after the earliest + /// change is included, found as a range of the session index; refreshing + /// an unchanged bucket writes nothing. + pub(crate) fn cache_successor_buckets( + &self, + floors: &BTreeMap>, + ) -> Result> { + let sql = format!( + "SELECT {CACHE_BUCKET_COLUMNS_SQL} FROM usage_events + WHERE {EVENT_SESSION_ID_SQL} = ?1 AND started_at >= ?2" + ); + let mut statement = self.conn.prepare(&sql)?; + let mut buckets = BTreeSet::new(); + for (session_id, floor) in floors { + let rows = + statement.query_map(params![session_id, floor.to_rfc3339()], cache_bucket_key)?; + for key in rows { + buckets.extend(key?.filter(|key| provider_has_cache_diagnostics(&key.provider))); + } + } + Ok(buckets) + } + + /// Rebuilds the daily summaries this report changes, once per detector + /// version: those of the covered providers, so history whose transcripts + /// are gone gains diagnostics too, and any other provider's that hold cache + /// writes, whose hit ratio now counts them. Summaries whose content is + /// unchanged are left alone, so nothing is queued for sync unless its + /// payload actually changed. + pub(crate) fn ensure_cache_health_rollups(&self) -> Result<()> { + let version = CACHE_HEALTH_VERSION.to_string(); + if self + .metadata_value(CACHE_HEALTH_ROLLUPS_METADATA_KEY)? + .as_deref() + == Some(version.as_str()) + { + return Ok(()); + } + let keys = self.cache_report_bucket_keys()?; + self.with_immediate_transaction(|| { + self.refresh_sync_rollups_for_keys(&keys)?; + self.set_metadata_value(CACHE_HEALTH_ROLLUPS_METADATA_KEY, &version) + }) + } + + fn cache_report_bucket_keys(&self) -> Result> { + let placeholders = sqlite_in_clause_placeholders(CACHE_DIAGNOSTIC_PROVIDERS.len()); + let sql = format!( + "SELECT {CACHE_BUCKET_COLUMNS_SQL} FROM usage_events + WHERE provider IN ({placeholders}) + OR (json_valid(payload) + AND json_extract(payload, '$.usage.cache_creation_tokens') > 0)" + ); + let mut statement = self.conn.prepare(&sql)?; + let rows = statement.query_map( + rusqlite::params_from_iter(CACHE_DIAGNOSTIC_PROVIDERS), + cache_bucket_key, + )?; + let mut keys = BTreeSet::new(); + for key in rows { + keys.extend(key?); + } + Ok(keys) + } + + fn analyzed_session<'memo>( + &self, + session_id: &str, + memo: &'memo mut CacheSessionMemo, + ) -> Result<&'memo AnalyzedSession> { + if !memo.sessions.contains_key(session_id) { + let events = self.cache_session_events(session_id)?; + let mut calls_by_event = AnalyzedSession::new(); + for call in analyze_and_price(&events) { + calls_by_event + .entry(call.event_id.0.clone()) + .or_default() + .push(call); + } + if memo.order.len() >= CACHE_SESSION_MEMO_CAPACITY { + if let Some(evicted) = memo.order.pop_front() { + memo.sessions.remove(&evicted); + } + } + memo.order.push_back(session_id.to_string()); + memo.sessions.insert(session_id.to_string(), calls_by_event); + } + Ok(&memo.sessions[session_id]) + } + + /// Every event of a session, in time order. + fn cache_session_events(&self, session_id: &str) -> Result> { + let sql = format!( + "SELECT payload FROM usage_events WHERE {EVENT_SESSION_ID_SQL} = ?1 + ORDER BY started_at, event_id" + ); + let mut statement = self.conn.prepare(&sql)?; + let rows = statement.query_map(params![session_id], |row| row.get::<_, String>(0))?; + let mut events = Vec::new(); + for row in rows { + events.push(serde_json::from_str(&row?)?); + } + Ok(events) + } + + /// Every event of a session, each marked with whether it starts inside the + /// report's period. The bounds compare as `events_in_period` compares them. + fn cache_session_events_in_period( + &self, + session_id: &str, + query: &CacheReportQuery, + ) -> Result> { + let sql = format!( + "SELECT payload, (?2 IS NULL OR started_at >= ?2) AND started_at <= ?3 + FROM usage_events WHERE {EVENT_SESSION_ID_SQL} = ?1 + ORDER BY started_at, event_id" + ); + let mut statement = self.conn.prepare(&sql)?; + let rows = statement.query_map( + params![ + session_id, + query.since.map(|since| since.to_rfc3339()), + query.until.to_rfc3339() + ], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, bool>(1)?)), + )?; + let mut events = Vec::new(); + for row in rows { + let (payload, in_period) = row?; + events.push((serde_json::from_str(&payload)?, in_period)); + } + Ok(events) + } + + /// Sessions of the covered providers with an event in the report's period + /// that its provider and session filters allow. + fn cache_report_sessions(&self, query: &CacheReportQuery) -> Result> { + let providers = CACHE_DIAGNOSTIC_PROVIDERS + .iter() + .filter(|provider| { + query + .provider + .as_deref() + .is_none_or(|filter| filter == **provider) + }) + .map(|provider| (*provider).to_string()) + .collect::>(); + if providers.is_empty() { + return Ok(Vec::new()); + } + let mut bounds = vec![query.until.to_rfc3339()]; + let since_sql = match query.since { + Some(since) => { + bounds.push(since.to_rfc3339()); + "AND started_at >= ?" + } + None => "", + }; + let sql = format!( + "SELECT DISTINCT {EVENT_SESSION_ID_SQL} FROM usage_events + WHERE started_at <= ? {since_sql} AND provider IN ({})", + sqlite_in_clause_placeholders(providers.len()) + ); + let mut statement = self.conn.prepare(&sql)?; + let rows = statement.query_map( + rusqlite::params_from_iter(bounds.iter().chain(&providers)), + |row| row.get::<_, Option>(0), + )?; + let mut sessions = Vec::new(); + for session_id in rows { + let Some(session_id) = session_id? else { + continue; + }; + if query + .session + .as_deref() + .is_none_or(|filter| session_id.starts_with(filter)) + { + sessions.push(session_id); + } + } + Ok(sessions) + } + + /// Events in the report's period from providers without diagnostics. + fn cache_report_other_events(&self, query: &CacheReportQuery) -> Result> { + if query + .provider + .as_deref() + .is_some_and(provider_has_cache_diagnostics) + { + return Ok(Vec::new()); + } + let mut bounds = vec![query.until.to_rfc3339()]; + let since_sql = match query.since { + Some(since) => { + bounds.push(since.to_rfc3339()); + "AND started_at >= ?" + } + None => "", + }; + let sql = format!( + "SELECT payload FROM usage_events + WHERE started_at <= ? {since_sql} AND provider NOT IN ({})", + sqlite_in_clause_placeholders(CACHE_DIAGNOSTIC_PROVIDERS.len()) + ); + let mut statement = self.conn.prepare(&sql)?; + let rows = statement.query_map( + rusqlite::params_from_iter( + bounds + .iter() + .map(String::as_str) + .chain(CACHE_DIAGNOSTIC_PROVIDERS.iter().copied()), + ), + |row| row.get::<_, String>(0), + )?; + let mut events = Vec::new(); + for row in rows { + events.push(serde_json::from_str(&row?)?); + } + Ok(events) + } +} + +/// A summary in the form a target stores it. Selection and the recorded +/// payload hash both see this form, so a receiver without `cache_health` +/// support is not re-sent summaries that differ only by it. +pub fn summary_for_sync_target(summary: UsageSummary, accepts_cache_health: bool) -> UsageSummary { + if accepts_cache_health { + summary + } else { + summary.without_cache_health() + } +} + +/// A summary that starts within the report's range. The dashboard's period +/// views count a summary the same way, including one that spans several days. +fn cache_report_includes_summary(query: &CacheReportQuery, summary: &UsageSummary) -> bool { + let (start, _) = summary_period_bounds(summary); + let in_range = query.since.is_none_or(|since| start >= since) && start <= query.until; + let provider_matches = query + .provider + .as_deref() + .is_none_or(|provider| summary.provider == provider); + let account_matches = + query + .account + .as_deref() + .is_none_or(|account| match summary.provider_account_id.as_ref() { + Some(id) => id.0 == account, + None => account == CACHE_REPORT_UNASSIGNED_ACCOUNT, + }); + in_range && provider_matches && account_matches +} + +fn cache_report_includes(query: &CacheReportQuery, event: &UsageEvent) -> bool { + let provider_matches = query + .provider + .as_deref() + .is_none_or(|provider| event.provider == provider); + let account_matches = + query + .account + .as_deref() + .is_none_or(|account| match event.provider_account_id.as_ref() { + Some(id) => id.0 == account, + None => account == CACHE_REPORT_UNASSIGNED_ACCOUNT, + }); + let session_matches = query + .session + .as_deref() + .is_none_or(|session| event.session.session_id.starts_with(session)); + provider_matches && account_matches && session_matches +} + +/// Classifies a session's calls and prices the reuse each suspected loss +/// missed. +fn analyze_and_price(events: &[UsageEvent]) -> Vec { + let by_id = events + .iter() + .map(|event| (event.event_id.0.as_str(), event)) + .collect::>(); + let mut calls = analyze_cache_calls(events); + for call in &mut calls { + let (Some(missed), Some(event)) = ( + call.verdict.missed_tokens, + by_id.get(call.event_id.0.as_str()), + ) else { + continue; + }; + call.verdict.missed_cost_micro_usd = missed_reuse_cost(event, call, missed); + } + calls +} + +/// The call's cost at API prices minus its cost had `missed` tokens been +/// cache reads. A lost cache is written again, so they come out of the call's +/// cache writes first, then out of its ordinary input. `None` when the model +/// has no known pricing. +fn missed_reuse_cost(event: &UsageEvent, call: &AnalyzedCacheCall, missed: u64) -> Option { + // A call inside an aggregated event has no split of its writes by cache + // lifetime, and is priced by its own model alone: the event's model + // identity, such as its provider model id, belongs to the turn's last call. + let single_call = event + .context + .as_ref() + .is_none_or(|context| context.calls.is_empty()); + let model = match call.model.as_deref() { + Some(name) if !single_call => Some(statsai_core::ModelInfo { + name: Some(name.to_string()), + normalized_name: Some(name.to_string()), + ..statsai_core::ModelInfo::default() + }), + _ => event.model.clone(), + }; + // One request, so per-request pricing such as long-context rates applies. + let actual = UsageCounts { + input_tokens: Some(call.input_tokens), + cache_read_tokens: Some(call.cache_read_tokens), + cache_creation_tokens: call.cache_creation_tokens, + cache_creation_5m_tokens: event.usage.cache_creation_5m_tokens.filter(|_| single_call), + cache_creation_1h_tokens: event.usage.cache_creation_1h_tokens.filter(|_| single_call), + requests: Some(1), + ..UsageCounts::default() + }; + let reused = usage_with_reads(&actual, missed); + let cost = |usage: &UsageCounts| { + statsai_pricing::estimate_cost_at(&call.provider, model.as_ref(), usage, &call.completed_at) + .estimated_api_equivalent_micro_usd + }; + let difference = cost(&actual)?.saturating_sub(cost(&reused)?); + Some(u64::try_from(difference).unwrap_or(0)) +} + +/// `usage` with `tokens` moved into cache reads from cache writes, keeping +/// the split by cache lifetime in proportion, and then from ordinary input. +fn usage_with_reads(usage: &UsageCounts, tokens: u64) -> UsageCounts { + let writes = usage.cache_creation_tokens.unwrap_or(0); + let from_writes = tokens.min(writes); + let from_input = (tokens - from_writes).min(usage.input_tokens.unwrap_or(0)); + let shrink = |part: Option| { + part.map(|part| { + let removed = u128::from(part) * u128::from(from_writes) / u128::from(writes.max(1)); + part.saturating_sub(u64::try_from(removed).unwrap_or(part)) + }) + }; + UsageCounts { + input_tokens: usage.input_tokens.map(|input| input - from_input), + cache_read_tokens: Some( + usage + .cache_read_tokens + .unwrap_or(0) + .saturating_add(from_writes + from_input), + ), + cache_creation_tokens: usage.cache_creation_tokens.map(|_| writes - from_writes), + cache_creation_5m_tokens: shrink(usage.cache_creation_5m_tokens), + cache_creation_1h_tokens: shrink(usage.cache_creation_1h_tokens), + ..usage.clone() + } +} + +/// The daily bucket of a row of [`CACHE_BUCKET_COLUMNS_SQL`], or `None` when +/// its start time does not parse. +fn cache_bucket_key(row: &rusqlite::Row<'_>) -> rusqlite::Result> { + let started_at = row.get::<_, String>(3)?; + let Ok(started_at) = DateTime::parse_from_rfc3339(&started_at) else { + return Ok(None); + }; + let project = row.get::<_, Option>(4)?; + let project = project + .as_deref() + .and_then(|project| serde_json::from_str::(project).ok()); + Ok(Some(SyncRollupBucketKey { + provider: row.get(0)?, + source_id: row.get(1)?, + provider_account_id: row.get(2)?, + day_key: started_at.with_timezone(&Utc).date_naive().to_string(), + project_key: sync_rollup_project_key(project.as_ref()), + })) +} diff --git a/crates/statsai-store/src/events.rs b/crates/statsai-store/src/events.rs index 92483f8..f15dc7e 100644 --- a/crates/statsai-store/src/events.rs +++ b/crates/statsai-store/src/events.rs @@ -5,6 +5,9 @@ use crate::dedupe::*; pub(crate) struct EventWriteDirty { pub(crate) buckets: BTreeSet, pub(crate) sessions: BTreeSet, + /// Earliest changed event per session, for providers with prompt-cache + /// diagnostics. Later calls in the session may need their verdicts redone. + pub(crate) cache_floors: BTreeMap>, } impl EventWriteDirty { @@ -17,11 +20,24 @@ impl EventWriteDirty { fn observe(&mut self, event: &UsageEvent) { self.buckets.insert(sync_rollup_bucket_key(event)); self.sessions.insert(event.session.session_id.clone()); + if statsai_core::provider_has_cache_diagnostics(&event.provider) { + self.observe_cache_floor(&event.session.session_id, event.session.started_at); + } + } + + fn observe_cache_floor(&mut self, session_id: &str, started_at: DateTime) { + self.cache_floors + .entry(session_id.to_string()) + .and_modify(|floor| *floor = (*floor).min(started_at)) + .or_insert(started_at); } pub(crate) fn extend(&mut self, other: Self) { self.buckets.extend(other.buckets); self.sessions.extend(other.sessions); + for (session_id, started_at) in other.cache_floors { + self.observe_cache_floor(&session_id, started_at); + } } } @@ -164,8 +180,10 @@ impl Store { }) } - fn refresh_event_rollups(&self, dirty: &EventWriteDirty) -> Result<()> { - self.refresh_sync_rollups_for_keys(&dirty.buckets)?; + pub(crate) fn refresh_event_rollups(&self, dirty: &EventWriteDirty) -> Result<()> { + let mut buckets = self.cache_successor_buckets(&dirty.cache_floors)?; + buckets.extend(dirty.buckets.iter().cloned()); + self.refresh_sync_rollups_for_keys(&buckets)?; self.refresh_session_rollups_for_keys(&dirty.sessions)?; Ok(()) } diff --git a/crates/statsai-store/src/lib.rs b/crates/statsai-store/src/lib.rs index 1ca46a8..fdb4aac 100644 --- a/crates/statsai-store/src/lib.rs +++ b/crates/statsai-store/src/lib.rs @@ -4,6 +4,7 @@ mod account_plan; mod accounts; mod activity; mod archive; +mod cache; mod code_changes; mod dedupe; mod events; @@ -49,18 +50,18 @@ use chrono::{DateTime, Datelike, NaiveDate, Utc}; use rusqlite::{params, Connection, OptionalExtension}; use serde::{Deserialize, Serialize}; use statsai_core::{ - daily_rollup_project_key, hash_text, micro_usd_to_cents_rounded, normalize_email, - normalize_provider_user_id, periods_overlap, project_contains_file_paths, + cache_hit_ratio, daily_rollup_project_key, hash_text, micro_usd_to_cents_rounded, + normalize_email, normalize_provider_user_id, periods_overlap, project_contains_file_paths, project_has_stable_identity, provider_account_id, provider_account_id_from_identity, sanitize_code_change_metric_for_sync, sanitize_summary_for_sync, semantic_event_fingerprint, source_account_assignment_id, subscription_id, summary_id, timestamp_in_period, AccountEvidenceSummaryV1, AccountPlanProjectionV1, ActivityCoverageV1, ActivityInvocationV1, - BillingPeriod, CodeChangeMetric, Confidence, CostAccumulator, CostInfo, DailyRollup, EventId, - EventSource, IdentitySource, LatencySource, MetricStats, ModelInfo, PrivacyInfo, PrivacyMode, - ProviderAccount, ProviderAccountId, SemanticFingerprintInput, SourceAccountAssignment, - SourceAccountAssignmentId, SourceId, SourceKind, SourceLocation, SourceVerificationMode, - Subscription, SubscriptionId, SubscriptionStatus, SummaryId, SummaryMetadata, - SummaryMetricTotals, SummaryMetrics, SummaryModelMetrics, SummaryModelUsage, + BillingPeriod, CacheHealthV1, CodeChangeMetric, Confidence, CostAccumulator, CostInfo, + DailyRollup, EventId, EventSource, IdentitySource, LatencySource, MetricStats, ModelInfo, + PrivacyInfo, PrivacyMode, ProviderAccount, ProviderAccountId, SemanticFingerprintInput, + SourceAccountAssignment, SourceAccountAssignmentId, SourceId, SourceKind, SourceLocation, + SourceVerificationMode, Subscription, SubscriptionId, SubscriptionStatus, SummaryId, + SummaryMetadata, SummaryMetricTotals, SummaryMetrics, SummaryModelMetrics, SummaryModelUsage, SyncAuthoritativeSnapshot, SyncBatch, TaskVerificationCursor, TaskVerificationId, UsageCounts, UsageEvent, UsageSummary, VerifiedSourceObservation, VerifiedSourceState, VerifiedSubscriptionState, PROVIDER_ACCOUNT_SCHEMA_VERSION, @@ -96,6 +97,8 @@ const INFERRED_SOURCE_OBSERVATION_HASH_PREFIX: &str = "inferred_source.v1:"; const VERIFIED_SOURCE_OBSERVATION_HASH_PREFIX: &str = "verified_source.v2:"; pub use archive::{ArchiveConversationSummary, ArchiveSearchHit, ArchiveStats, ArchiveWriteResult}; +pub(crate) use cache::CacheSessionMemo; +pub use cache::{summary_for_sync_target, CacheReportQuery, CACHE_REPORT_UNASSIGNED_ACCOUNT}; pub use code_changes::CodeChangeRefreshReport; pub use privacy::{ FilteredConversationMetadata, FilteredConversationRecord, PrivacyDatasetStatus, diff --git a/crates/statsai-store/src/migrations/mod.rs b/crates/statsai-store/src/migrations/mod.rs index e944d3c..a769d63 100644 --- a/crates/statsai-store/src/migrations/mod.rs +++ b/crates/statsai-store/src/migrations/mod.rs @@ -8,7 +8,7 @@ mod v2; pub(crate) use v1::*; pub(crate) use v2::*; -pub const CURRENT_SCHEMA_VERSION: i64 = 29; +pub const CURRENT_SCHEMA_VERSION: i64 = 30; pub fn migrate(conn: &Connection) -> Result<()> { if let Some(current) = existing_schema_version(conn)? { @@ -154,6 +154,7 @@ fn apply_migration(conn: &Connection, version: i64) -> Result<()> { 27 => apply_migration_027(conn), 28 => apply_migration_028(conn), 29 => apply_migration_029(conn), + 30 => apply_migration_030(conn), _ => bail!("unsupported schema migration version {version}"), } } @@ -291,7 +292,10 @@ pub fn schema_version(conn: &Connection) -> Result { #[cfg(test)] mod tests { use super::*; - use crate::{EVENT_CONVERSATION_HASH_SQL, EVENT_SOURCE_FILE_HASH_SQL, QUOTA_PLAN_TYPE_SQL}; + use crate::{ + EVENT_CONVERSATION_HASH_SQL, EVENT_SESSION_ID_SQL, EVENT_SOURCE_FILE_HASH_SQL, + QUOTA_PLAN_TYPE_SQL, + }; use rusqlite::Connection; #[test] @@ -356,6 +360,40 @@ mod tests { &conn, "usage_events_provider_account_started_idx" )); + assert!(index_exists(&conn, "usage_events_session_started_idx")); + assert!(!index_exists(&conn, "usage_events_session_idx")); + } + + /// Session reads come back in time order straight from the index, and the + /// calls after a changed event are a range of it, so neither sorts nor + /// scans the session. + #[test] + fn session_reads_are_ordered_ranges_of_the_session_index() { + let conn = Connection::open_in_memory().expect("open in-memory database"); + migrate(&conn).expect("migrate fresh database"); + + for (label, sql) in [ + ( + "read a session in order", + format!( + "SELECT payload FROM usage_events WHERE {EVENT_SESSION_ID_SQL} = ?1 + ORDER BY started_at, event_id" + ), + ), + ( + "find the events after a changed one", + format!( + "SELECT provider FROM usage_events + WHERE {EVENT_SESSION_ID_SQL} = ?1 AND started_at >= ?2" + ), + ), + ] { + let plan = query_plan(&conn, &sql); + assert!( + plan.contains("usage_events_session_started_idx") && !plan.contains("TEMP B-TREE"), + "{label} does not read the session index in order: {plan}" + ); + } } /// The scan filters are only fast while SQLite can match them to the indexes diff --git a/crates/statsai-store/src/migrations/v2.rs b/crates/statsai-store/src/migrations/v2.rs index 021e46f..67c475c 100644 --- a/crates/statsai-store/src/migrations/v2.rs +++ b/crates/statsai-store/src/migrations/v2.rs @@ -684,3 +684,22 @@ pub(crate) fn apply_migration_029(conn: &Connection) -> Result<()> { )?; Ok(()) } + +/// Orders each session's events by time in the session index. +/// +/// The prompt-cache detector judges each call against the one before it in +/// its session, so it reads sessions whole, and after a late arrival or a +/// correction it looks up the session's events at or after the changed one. +/// Leading with the session and then `started_at` serves both as index range +/// scans, and gives session reads their order without a sort. It replaces +/// `usage_events_session_idx`, whose session lookups it also covers. +pub(crate) fn apply_migration_030(conn: &Connection) -> Result<()> { + conn.execute_batch(&format!( + r#" + CREATE INDEX IF NOT EXISTS usage_events_session_started_idx + ON usage_events ({EVENT_SESSION_ID_SQL}, started_at, event_id); + DROP INDEX IF EXISTS usage_events_session_idx; + "#, + ))?; + Ok(()) +} diff --git a/crates/statsai-store/src/pricing/tests/repricing.rs b/crates/statsai-store/src/pricing/tests/repricing.rs index 20b9f8b..1f5ca3b 100644 --- a/crates/statsai-store/src/pricing/tests/repricing.rs +++ b/crates/statsai-store/src/pricing/tests/repricing.rs @@ -384,7 +384,7 @@ fn daily_rollups_are_unused_by_report_and_sync_paths() { assert_eq!(events[0].cost, expected_review_cost(started_at)); // Pin: bump this when CURRENT_SCHEMA_VERSION changes, after confirming // the daily_rollups table is still unused by report/sync/snapshot. - assert_eq!(CURRENT_SCHEMA_VERSION, 29); + assert_eq!(CURRENT_SCHEMA_VERSION, 30); } #[test] diff --git a/crates/statsai-store/src/pricing/tests/support.rs b/crates/statsai-store/src/pricing/tests/support.rs index 828b453..02abc63 100644 --- a/crates/statsai-store/src/pricing/tests/support.rs +++ b/crates/statsai-store/src/pricing/tests/support.rs @@ -87,6 +87,7 @@ pub(super) fn test_event( }, created_at: started_at, imported_at: started_at, + context: None, } } diff --git a/crates/statsai-store/src/quota/tests/support.rs b/crates/statsai-store/src/quota/tests/support.rs index c4e634d..c681f3f 100644 --- a/crates/statsai-store/src/quota/tests/support.rs +++ b/crates/statsai-store/src/quota/tests/support.rs @@ -172,5 +172,6 @@ pub(super) fn sample_usage_event( }, created_at: started_at, imported_at: started_at, + context: None, } } diff --git a/crates/statsai-store/src/rollups/build.rs b/crates/statsai-store/src/rollups/build.rs index 30bb3a5..0908e36 100644 --- a/crates/statsai-store/src/rollups/build.rs +++ b/crates/statsai-store/src/rollups/build.rs @@ -48,7 +48,10 @@ pub(crate) fn event_with_valid_project(event: &UsageEvent) -> UsageEvent { event } -pub(crate) fn build_sync_rollup_summary(events: &[UsageEvent]) -> UsageSummary { +pub(crate) fn build_sync_rollup_summary( + events: &[UsageEvent], + cache_health: Option, +) -> UsageSummary { let first = events.first().expect("rollup bucket must contain events"); // Events arrive oldest first. A bucket can now span a repository rename, so // project metadata comes from the newest event: it names the remote the @@ -177,14 +180,12 @@ pub(crate) fn build_sync_rollup_summary(events: &[UsageEvent]) -> UsageSummary { tracked_reasoning_tokens.saturating_add(event.usage.reasoning_tokens.unwrap_or(0)); } - let prompt_tokens = event - .usage - .input_tokens - .unwrap_or(0) - .saturating_add(event.usage.cache_read_tokens.unwrap_or(0)); - if prompt_tokens > 0 { - cache_hit_ratio_values - .push(event.usage.cache_read_tokens.unwrap_or(0) as f64 / prompt_tokens as f64); + if let Some(ratio) = cache_hit_ratio( + event.usage.input_tokens.unwrap_or(0), + event.usage.cache_creation_tokens.unwrap_or(0), + event.usage.cache_read_tokens.unwrap_or(0), + ) { + cache_hit_ratio_values.push(ratio); } let generated_tokens = event .usage @@ -318,6 +319,7 @@ pub(crate) fn build_sync_rollup_summary(events: &[UsageEvent]) -> UsageSummary { user_messages: (user_messages > 0).then_some(user_messages), assistant_messages: (assistant_messages > 0).then_some(assistant_messages), developer_messages: (developer_messages > 0).then_some(developer_messages), + cache_health, }); let total_sessions = (!session_ids.is_empty()).then_some(session_ids.len() as u64); let total_messages_metadata = summary_metrics @@ -424,23 +426,7 @@ fn percentile_nearest_rank(values: &[f64], percentile: f64) -> Option { } fn summary_metrics_or_none(metrics: SummaryMetrics) -> Option { - let has_metrics = metrics.active_seconds.is_some() - || metrics.tracked_requests.is_some() - || metrics.tracked_output_tokens.is_some() - || metrics.tracked_reasoning_tokens.is_some() - || metrics.latency_ms.is_some() - || metrics.time_to_first_token_ms.is_some() - || metrics.generated_tps.is_some() - || metrics.visible_tps.is_some() - || metrics.overall_generated_tps.is_some() - || metrics.overall_visible_tps.is_some() - || metrics.cache_hit_ratio.is_some() - || metrics.reasoning_share.is_some() - || metrics.total_messages.is_some() - || metrics.user_messages.is_some() - || metrics.assistant_messages.is_some() - || metrics.developer_messages.is_some(); - has_metrics.then_some(metrics) + (!metrics.is_empty()).then_some(metrics) } #[derive(Debug, Default)] diff --git a/crates/statsai-store/src/rollups/mod.rs b/crates/statsai-store/src/rollups/mod.rs index 38807a1..339934f 100644 --- a/crates/statsai-store/src/rollups/mod.rs +++ b/crates/statsai-store/src/rollups/mod.rs @@ -84,7 +84,7 @@ impl Store { if stale_count > 0 { self.rebuild_sync_rollups()?; } - Ok(()) + self.ensure_cache_health_rollups() } pub fn sync_rollup_period_stats(&self, cutoff_day: NaiveDate) -> Result { @@ -257,10 +257,16 @@ impl Store { } fn pending_sync_rollup_counts(&self, sink: &str, target: &str) -> Result<(u64, u64)> { + let accepts_cache_health = self.sync_target_accepts_cache_health(sink, target)?; let rollups = self .all_sync_rollup_summaries()? .into_iter() - .map(sanitize_summary_for_default_http_sync) + .map(|summary| { + summary_for_sync_target( + sanitize_summary_for_default_http_sync(summary), + accepts_cache_health, + ) + }) .collect::>(); let pending = self.pending_summaries_for_sync(sink, target, &rollups)?; let days = collect_pending_summary_days(pending.iter()); @@ -269,7 +275,12 @@ impl Store { pub fn reconcile_sync_rollup_dirty_flags(&self, sink: &str, target: &str) -> Result { self.ensure_current_sync_rollup_versions()?; - let summaries = self.all_sync_rollup_summaries()?; + let accepts_cache_health = self.sync_target_accepts_cache_health(sink, target)?; + let summaries = self + .all_sync_rollup_summaries()? + .into_iter() + .map(|summary| summary_for_sync_target(summary, accepts_cache_health)) + .collect::>(); self.with_immediate_transaction(|| { self.reconcile_sync_rollup_dirty_flags_in_transaction(sink, target, &summaries) }) @@ -470,16 +481,21 @@ impl Store { &self, keys: &BTreeSet, ) -> Result { + let mut memo = CacheSessionMemo::default(); let mut refreshed = 0u64; for key in keys { - if self.refresh_sync_rollup_for_key(key)? { + if self.refresh_sync_rollup_for_key(key, &mut memo)? { refreshed += 1; } } Ok(refreshed) } - fn refresh_sync_rollup_for_key(&self, key: &SyncRollupBucketKey) -> Result { + fn refresh_sync_rollup_for_key( + &self, + key: &SyncRollupBucketKey, + memo: &mut CacheSessionMemo, + ) -> Result { let events = self.sync_rollup_events(key)?; if events.is_empty() { let deleted = self.conn.execute( @@ -489,7 +505,8 @@ impl Store { return Ok(deleted > 0); } - let summary = build_sync_rollup_summary(&events); + let cache_health = self.bucket_cache_health(key, &events, memo)?; + let summary = build_sync_rollup_summary(&events, cache_health); let payload = serde_json::to_string(&summary)?; let payload_hash = summary_sync_payload_hash(&summary)?; let existing: Option<(String, i64)> = self diff --git a/crates/statsai-store/src/sync_state/mod.rs b/crates/statsai-store/src/sync_state/mod.rs index a19be32..28018da 100644 --- a/crates/statsai-store/src/sync_state/mod.rs +++ b/crates/statsai-store/src/sync_state/mod.rs @@ -563,7 +563,12 @@ impl Store { .filter(|summary| is_daily_rollup_summary(summary)) .map(|summary| summary.summary_id.clone()) .collect(); - let rollup_summaries = self.all_sync_rollup_summaries()?; + let accepts_cache_health = self.sync_target_accepts_cache_health(sink, target)?; + let rollup_summaries = self + .all_sync_rollup_summaries()? + .into_iter() + .map(|summary| summary_for_sync_target(summary, accepts_cache_health)) + .collect::>(); let task_verification_cursor = Self::sync_batch_task_verification_cursor(batch); self.with_immediate_transaction(|| { diff --git a/crates/statsai-store/src/sync_state/pending.rs b/crates/statsai-store/src/sync_state/pending.rs index 8c38c2b..d0bcc85 100644 --- a/crates/statsai-store/src/sync_state/pending.rs +++ b/crates/statsai-store/src/sync_state/pending.rs @@ -421,10 +421,16 @@ impl Store { target: &str, include_projects: bool, ) -> Result> { + let accepts_cache_health = self.sync_target_accepts_cache_health("http", target)?; let rollups = self .all_sync_rollup_summaries()? .into_iter() - .map(|summary| sanitize_summary_for_http_sync(summary, include_projects)) + .map(|summary| { + summary_for_sync_target( + sanitize_summary_for_http_sync(summary, include_projects), + accepts_cache_health, + ) + }) .collect::>(); self.pending_summaries_for_sync("http", target, &rollups) } @@ -443,17 +449,28 @@ impl Store { device_id: &str, include_projects: bool, ) -> Result { + let accepts_cache_health = self.sync_target_accepts_cache_health("http", target)?; let current_rollups = self .all_sync_rollup_summaries()? .into_iter() - .map(|summary| sanitize_summary_for_http_sync(summary, include_projects)) + .map(|summary| { + summary_for_sync_target( + sanitize_summary_for_http_sync(summary, include_projects), + accepts_cache_health, + ) + }) .collect::>(); let rollups = self.pending_summaries_for_sync("http", target, ¤t_rollups)?; let current_passthrough_summaries = self .summaries()? .into_iter() .filter(is_http_rollup_passthrough_summary) - .map(|summary| sanitize_summary_for_http_sync(summary, include_projects)) + .map(|summary| { + summary_for_sync_target( + sanitize_summary_for_http_sync(summary, include_projects), + accepts_cache_health, + ) + }) .collect::>(); let passthrough_summaries = self.pending_summaries_for_sync("http", target, ¤t_passthrough_summaries)?; diff --git a/crates/statsai-store/src/tests/cache.rs b/crates/statsai-store/src/tests/cache.rs new file mode 100644 index 0000000..4ca1c2f --- /dev/null +++ b/crates/statsai-store/src/tests/cache.rs @@ -0,0 +1,636 @@ +use super::support::*; +use super::*; +use statsai_core::{ + CacheHealthV1, CallContext, EventId, IdentitySource, ModelCall, CACHE_HEALTH_VERSION, +}; + +fn claude_source() -> SourceLocation { + SourceLocation::local_adapter( + "claude_code", + "test", + "0", + Path::new("/fixture/claude"), + LocationOrigin::Configured, + ) +} + +fn at(day: u32, hour: u32, minute: u32) -> chrono::DateTime { + Utc.with_ymd_and_hms(2026, 7, day, hour, minute, 0) + .single() + .expect("time") +} + +/// One Claude Code call in session `session_cache`, transcript `main`. +fn call( + source: &SourceLocation, + index: u64, + time: chrono::DateTime, + input: u64, + read: u64, + write: u64, +) -> UsageEvent { + let mut event = test_store_event(source, time, &format!("call-{index}")); + event.provider = "claude_code".to_string(); + event.event_id = EventId(format!("event_cache_{index:03}")); + event.session.session_id = "session_cache".to_string(); + event.usage = UsageCounts { + input_tokens: Some(input), + cache_read_tokens: Some(read), + cache_creation_tokens: Some(write), + output_tokens: Some(10), + total_tokens: Some(input + read + write + 10), + requests: Some(1), + ..UsageCounts::default() + }; + event.parse_evidence = Some(ParseEvidence { + event_key_version: "test".to_string(), + source_file_path_hash: Some("main".to_string()), + source_line_number: Some(index), + source_record_id: None, + model_inferred: false, + timestamp_inferred: false, + account_identity_source: IdentitySource::Unresolved, + }); + event +} + +fn store_with_source() -> (Store, SourceLocation) { + let store = Store::in_memory().expect("store"); + let source = claude_source(); + store.upsert_source(&source).expect("source"); + (store, source) +} + +fn health_by_day(store: &Store) -> BTreeMap { + store + .all_sync_rollup_summaries() + .expect("summaries") + .into_iter() + .filter_map(|summary| { + let day = summary.period_start?.date_naive().to_string(); + Some((day, summary.metrics?.cache_health?)) + }) + .collect() +} + +fn payload_hashes(store: &Store) -> BTreeMap { + let mut statement = store + .conn + .prepare("SELECT summary_id, payload_hash, updated_at FROM sync_rollups") + .expect("statement"); + statement + .query_map([], |row| Ok((row.get(0)?, (row.get(1)?, row.get(2)?)))) + .expect("rows") + .collect::>() + .expect("hashes") +} + +#[test] +fn daily_summaries_carry_cache_health_for_covered_providers_only() { + let (store, source) = store_with_source(); + let mut events = vec![ + call(&source, 1, at(1, 10, 0), 10, 0, 30_000), + call(&source, 2, at(1, 10, 1), 10, 30_000, 500), + call(&source, 3, at(1, 10, 20), 10, 0, 30_600), + ]; + let mut cursor = test_store_event(&source, at(1, 11, 0), "cursor-row"); + cursor.provider = "cursor".to_string(); + events.push(cursor); + store.insert_events(&events).expect("insert"); + + let summaries = store.all_sync_rollup_summaries().expect("summaries"); + let claude = summaries + .iter() + .find(|summary| summary.provider == "claude_code") + .expect("claude summary"); + let health = claude + .metrics + .as_ref() + .and_then(|metrics| metrics.cache_health.as_ref()) + .expect("cache health"); + assert_eq!(health.version, CACHE_HEALTH_VERSION); + assert_eq!( + (health.requests, health.first_observed, health.first_cold), + (3, 1, 1) + ); + assert_eq!( + (health.comparable, health.losses, health.full_losses), + (2, 1, 1) + ); + assert_eq!(health.gap_comparable[1], 1); + assert_eq!(health.gap_losses[4], 1); + // Writes count in the denominator of the per-call ratio. + assert_eq!( + claude + .metrics + .as_ref() + .and_then(|metrics| metrics.cache_hit_ratio.as_ref()) + .and_then(|ratio| ratio.max), + Some(30_000.0 / 30_510.0) + ); + + let cursor = summaries + .iter() + .find(|summary| summary.provider == "cursor") + .expect("cursor summary"); + assert!(cursor + .metrics + .as_ref() + .is_none_or(|metrics| metrics.cache_health.is_none())); + assert!( + serde_json::to_string(claude).expect("json").len() + - serde_json::to_string(&claude.clone().without_cache_health()) + .expect("json") + .len() + < 2_048, + "the synced object stays under 2 KiB" + ); +} + +#[test] +fn a_predecessor_before_midnight_is_compared_with() { + let (store, source) = store_with_source(); + store + .insert_events(&[ + call(&source, 1, at(1, 23, 50), 10, 0, 50_000), + call(&source, 2, at(1, 23, 58), 10, 50_000, 100), + call(&source, 3, at(2, 0, 2), 10, 0, 50_100), + ]) + .expect("insert"); + + let health = health_by_day(&store); + assert_eq!(health["2026-07-01"].first_observed, 1); + let next_day = &health["2026-07-02"]; + assert_eq!( + next_day.first_observed, 0, + "the previous call is yesterday's" + ); + assert_eq!((next_day.comparable, next_day.losses), (1, 1)); + assert_eq!(next_day.gap_losses[1], 1); +} + +#[test] +fn a_late_predecessor_refreshes_the_later_day() { + let (store, source) = store_with_source(); + store + .insert_events(&[call(&source, 3, at(2, 0, 2), 10, 0, 50_100)]) + .expect("later call"); + assert_eq!(health_by_day(&store)["2026-07-02"].first_observed, 1); + let before = payload_hashes(&store); + + store + .insert_events(&[ + call(&source, 1, at(1, 23, 50), 10, 0, 50_000), + call(&source, 2, at(1, 23, 58), 10, 50_000, 100), + ]) + .expect("late arrivals"); + + let health = health_by_day(&store); + assert_eq!(health["2026-07-02"].first_observed, 0); + assert_eq!(health["2026-07-02"].losses, 1); + let after = payload_hashes(&store); + let changed = before + .iter() + .filter(|(id, (hash, _))| after.get(*id).is_some_and(|(next, _)| next != hash)) + .count(); + assert_eq!(changed, 1, "the later day's summary was rebuilt"); +} + +#[test] +fn a_corrected_predecessor_changes_the_verdict_after_it() { + let (store, source) = store_with_source(); + store + .insert_events(&[ + call(&source, 1, at(1, 23, 58), 10, 50_000, 100), + call(&source, 2, at(2, 0, 2), 10, 0, 50_100), + ]) + .expect("insert"); + assert_eq!(health_by_day(&store)["2026-07-02"].losses, 1); + + // The predecessor turns out to have cached almost nothing. + store + .insert_event(&call(&source, 1, at(1, 23, 58), 50_100, 0, 10)) + .expect("correction"); + let next_day = &health_by_day(&store)["2026-07-02"]; + assert_eq!(next_day.losses, 0); + assert_eq!(next_day.below_threshold, 1); +} + +#[test] +fn sessions_split_by_sub_agent_are_compared_within_their_stream() { + let (store, source) = store_with_source(); + let mut agent = call(&source, 2, at(1, 10, 1), 0, 0, 20_000); + agent.context = Some(CallContext { + agent_hash: Some("agent".to_string()), + ..CallContext::default() + }); + store + .insert_events(&[ + call(&source, 1, at(1, 10, 0), 0, 100_000, 1_000), + agent, + call(&source, 3, at(1, 10, 2), 0, 101_000, 1_000), + ]) + .expect("insert"); + let health = &health_by_day(&store)["2026-07-01"]; + assert_eq!((health.first_observed, health.losses), (2, 0)); +} + +#[test] +fn the_one_time_refresh_adds_cache_health_once_and_leaves_other_summaries_alone() { + let (store, source) = store_with_source(); + let mut cursor = test_store_event(&source, at(1, 11, 0), "cursor-row"); + cursor.provider = "cursor".to_string(); + // Another provider's day with cache writes, whose hit ratio now counts them. + let mut opencode = call(&source, 3, at(1, 12, 0), 100, 100, 100); + opencode.provider = "opencode".to_string(); + opencode.event_id = EventId("event_opencode".to_string()); + opencode.session.session_id = "session_opencode".to_string(); + store + .insert_events(&[ + call(&source, 1, at(1, 10, 0), 10, 0, 30_000), + call(&source, 2, at(1, 10, 1), 10, 30_000, 500), + cursor, + opencode, + ]) + .expect("insert"); + store + .all_sync_rollup_summaries() + .expect("marker set on first read"); + let current = payload_hashes(&store); + + // A store written before this version: no cache_health, hit ratios + // without cache writes, and no marker. + for summary in store.all_sync_rollup_summaries().expect("summaries") { + let mut legacy = summary.without_cache_health(); + if legacy.provider == "opencode" { + let metrics = legacy.metrics.as_mut().expect("metrics"); + let ratio = metrics.cache_hit_ratio.as_mut().expect("hit ratio"); + ratio.p50 = Some(0.5); + } + store + .conn + .execute( + "UPDATE sync_rollups SET payload = ?2, payload_hash = ?3, updated_at = 'legacy', dirty = 0 + WHERE summary_id = ?1", + params![ + &legacy.summary_id.0, + serde_json::to_string(&legacy).expect("json"), + summary_sync_payload_hash(&legacy).expect("hash") + ], + ) + .expect("legacy row"); + } + store + .conn + .execute( + "DELETE FROM local_metadata WHERE key = 'sync_rollups.cache_health_version'", + [], + ) + .expect("clear marker"); + + assert_eq!(health_by_day(&store).len(), 1, "the refresh ran"); + let refreshed = payload_hashes(&store); + for (id, (hash, _)) in &refreshed { + assert_eq!(hash, ¤t[id].0, "content matches a fresh build"); + } + let untouched = refreshed + .values() + .filter(|(_, updated_at)| updated_at == "legacy") + .count(); + assert_eq!(untouched, 1, "the cursor summary was not rewritten"); + + // Running again changes nothing. + store.all_sync_rollup_summaries().expect("second read"); + assert_eq!(payload_hashes(&store), refreshed); +} + +#[test] +fn the_report_agrees_with_the_daily_summaries() { + let (store, source) = store_with_source(); + store + .insert_events(&[ + call(&source, 1, at(1, 23, 50), 10, 0, 50_000), + call(&source, 2, at(1, 23, 58), 10, 50_000, 100), + call(&source, 3, at(2, 0, 2), 10, 0, 50_100), + call(&source, 4, at(2, 1, 30), 10, 50_110, 200), + ]) + .expect("insert"); + let mut query = + CacheReportQuery::for_range(Some("2026-07-02"), Some("2026-07-02"), at(3, 0, 0)) + .expect("query"); + query.details = true; + let report = store.cache_report(&query, &Utc).expect("report"); + + let daily = &health_by_day(&store)["2026-07-02"]; + assert_eq!(&report.diagnostics, daily); + assert_eq!(report.totals.requests, 2); + assert_eq!(report.details.as_ref().map(Vec::len), Some(2)); + assert_eq!(report.days.len(), 1); + + let mut filtered = query.clone(); + filtered.account = Some("unassigned".to_string()); + filtered.session = Some("session_ca".to_string()); + assert_eq!( + store + .cache_report(&filtered, &Utc) + .expect("report") + .totals + .requests, + 2 + ); + filtered.provider = Some("codex".to_string()); + assert_eq!( + store + .cache_report(&filtered, &Utc) + .expect("report") + .totals + .requests, + 0 + ); +} + +#[test] +fn the_report_counts_other_providers_in_its_token_totals_only() { + let (store, source) = store_with_source(); + let other_source = SourceLocation::local_adapter( + "opencode", + "test", + "0", + Path::new("/fixture/opencode"), + LocationOrigin::Configured, + ); + store.upsert_source(&other_source).expect("source"); + let mut other = test_store_event(&other_source, at(2, 9, 0), "other"); + other.provider = "opencode".to_string(); + other.usage.requests = Some(1); + store + .insert_events(&[ + call(&source, 1, at(1, 23, 50), 10, 0, 50_000), + call(&source, 2, at(2, 0, 2), 10, 50_000, 100), + other, + ]) + .expect("insert"); + let query = CacheReportQuery::for_range(Some("2026-07-02"), Some("2026-07-02"), at(3, 0, 0)) + .expect("query"); + + let report = store.cache_report(&query, &Utc).expect("report"); + assert_eq!(report.totals.requests, 2); + assert_eq!(report.diagnostics.requests, 1); + assert_eq!(report.diagnostics.comparable, 1); + + let mut other_only = query.clone(); + other_only.provider = Some("opencode".to_string()); + let report = store.cache_report(&other_only, &Utc).expect("report"); + assert_eq!(report.totals.requests, 1); + assert_eq!(report.diagnostics.requests, 0); +} + +#[test] +fn a_receiver_without_cache_health_is_not_offered_summaries_that_differ_only_by_it() { + let (store, source) = store_with_source(); + store + .insert_events(&[ + call(&source, 1, at(1, 10, 0), 10, 0, 30_000), + call(&source, 2, at(1, 10, 1), 10, 30_000, 500), + ]) + .expect("insert"); + let target = "https://receiver.test/api/sync/batches"; + assert!(!store + .sync_target_accepts_cache_health("http", target) + .expect("default")); + assert!(store + .sync_target_accepts_cache_health("file", "out.json") + .expect("local sink")); + + let stripped = store + .all_sync_rollup_summaries() + .expect("summaries") + .into_iter() + .map(|summary| { + summary_for_sync_target(sanitize_summary_for_default_http_sync(summary), false) + }) + .collect::>(); + assert!(stripped.iter().all(|summary| summary + .metrics + .as_ref() + .is_none_or(|metrics| metrics.cache_health.is_none()))); + store + .record_summaries_synced("http", target, &stripped) + .expect("record"); + assert_eq!( + store + .pending_http_sync_summary_counts_with_projects(target, "device", false) + .expect("pending") + .rollups, + 0 + ); + + store + .record_sync_target_cache_health_support("http", target, true) + .expect("support"); + assert_eq!( + store + .pending_http_sync_summary_counts_with_projects(target, "device", false) + .expect("pending") + .rollups, + 1, + "support turning on queues the diagnostics once" + ); +} + +#[test] +fn the_report_counts_a_multi_day_summary_by_its_start_day() { + let (store, source) = store_with_source(); + let backfill = |key: &str, start: chrono::DateTime, reads: u64| { + let mut summary = test_store_summary(&source, start, reads); + summary.summary_id = summary_id(&source.provider, &source.source_id, key); + summary.source.source_kind = SourceKind::Manual; + summary.metadata.summary_format = "manual_period_summary".to_string(); + summary.period_start = Some(start); + summary.period_end = Some(start + chrono::Duration::days(5)); + summary.usage = UsageCounts { + cache_read_tokens: Some(reads), + total_tokens: Some(reads), + ..UsageCounts::default() + }; + summary + }; + store + .upsert_summary(&backfill("starts-inside", at(3, 0, 0), 1_000)) + .expect("summary"); + store + .upsert_summary(&backfill("starts-before", at(1, 0, 0), 50_000)) + .expect("summary"); + let query = CacheReportQuery::for_range(Some("2026-07-02"), Some("2026-07-09"), at(10, 0, 0)) + .expect("query"); + + let report = store.cache_report(&query, &Utc).expect("report"); + assert_eq!(report.totals.cache_read_tokens, 1_000); +} + +#[test] +fn missed_reuse_is_priced_as_rewritten_context_minus_cache_reads() { + let (store, source) = store_with_source(); + let model = |event: &mut UsageEvent, name: &str| { + event.model = Some(statsai_core::ModelInfo { + normalized_name: Some(name.to_string()), + ..statsai_core::ModelInfo::default() + }); + }; + let mut events = vec![ + call(&source, 1, at(1, 9, 0), 10, 0, 100_000), + // An hour and more later the context is written again, with the + // one-hour lifetime: 100,000 tokens at twice the input rate. + call(&source, 2, at(1, 10, 10), 10, 0, 100_000), + ]; + events[1].usage.cache_creation_1h_tokens = Some(100_000); + for event in &mut events { + model(event, "claude-sonnet-4-5"); + } + store.insert_events(&events).expect("insert"); + let query = CacheReportQuery::for_range(Some("2026-07-01"), Some("2026-07-01"), at(2, 0, 0)) + .expect("query"); + + let diagnostics = store + .cache_report(&query, &Utc) + .expect("report") + .diagnostics; + assert_eq!(diagnostics.missed_tokens, 100_000); + // $6.00 per million written for an hour, less $0.30 per million read. + assert_eq!(diagnostics.missed_cost_micro_usd, 570_000); + assert_eq!(diagnostics.unpriced_losses, 0); + assert_eq!( + health_by_day(&store)["2026-07-01"].missed_cost_micro_usd, + 570_000 + ); + + // A model without known pricing is counted, not priced. + let (store, source) = store_with_source(); + let mut events = vec![ + call(&source, 1, at(1, 9, 0), 10, 0, 100_000), + call(&source, 2, at(1, 10, 10), 10, 0, 100_000), + ]; + for event in &mut events { + model(event, "unpriced-model"); + } + store.insert_events(&events).expect("insert"); + let diagnostics = store + .cache_report(&query, &Utc) + .expect("report") + .diagnostics; + assert_eq!( + ( + diagnostics.losses, + diagnostics.unpriced_losses, + diagnostics.missed_cost_micro_usd + ), + (1, 1, 0) + ); +} + +#[test] +fn an_event_sharing_a_session_with_another_provider_is_counted_once() { + let (store, source) = store_with_source(); + let other_source = SourceLocation::local_adapter( + "opencode", + "test", + "0", + Path::new("/fixture/opencode"), + LocationOrigin::Configured, + ); + store.upsert_source(&other_source).expect("source"); + let mut other = test_store_event(&other_source, at(2, 9, 0), "other"); + other.provider = "opencode".to_string(); + other.session.session_id = "session_cache".to_string(); + other.usage.requests = Some(1); + store + .insert_events(&[call(&source, 1, at(2, 8, 0), 10, 0, 50_000), other]) + .expect("insert"); + let query = CacheReportQuery::for_range(Some("2026-07-02"), Some("2026-07-02"), at(3, 0, 0)) + .expect("query"); + + let report = store.cache_report(&query, &Utc).expect("report"); + assert_eq!(report.totals.requests, 2); + assert_eq!( + report + .providers + .iter() + .map(|row| (row.provider.as_str(), row.totals.requests)) + .collect::>(), + [("claude_code", 1), ("opencode", 1)] + ); +} + +#[test] +fn missed_reuse_is_priced_per_request_and_by_the_calls_own_model() { + let codex_source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/fixture/codex"), + LocationOrigin::Configured, + ); + let codex = |index: u64, minutes: u32, input: u64, read: u64| { + let mut event = call(&codex_source, index, at(1, 9, minutes), input, read, 0); + event.provider = "codex".to_string(); + event.usage.cache_creation_tokens = None; + event.model = Some(statsai_core::ModelInfo { + normalized_name: Some("gpt-6-astra".to_string()), + provider_model_id: Some("gpt-6-astra".to_string()), + ..statsai_core::ModelInfo::default() + }); + event + }; + let query = CacheReportQuery::for_range(Some("2026-07-01"), Some("2026-07-01"), at(2, 0, 0)) + .expect("query"); + + // A prompt over 272k tokens is priced at the long-context rates: $20 per + // million of input and $2 per million of cached input, not $10 and $1. + let store = Store::in_memory().expect("store"); + store.upsert_source(&codex_source).expect("source"); + store + .insert_events(&[codex(1, 0, 10, 300_000), codex(2, 1, 300_010, 0)]) + .expect("insert"); + let diagnostics = store + .cache_report(&query, &Utc) + .expect("report") + .diagnostics; + assert_eq!(diagnostics.missed_tokens, 300_000); + assert_eq!(diagnostics.missed_cost_micro_usd, 5_400_000); + + // A turn's call on a model without pricing stays unpriced, even though + // the turn's own model is priced. + let store = Store::in_memory().expect("store"); + store.upsert_source(&codex_source).expect("source"); + let mut turn = codex(1, 0, 50_010, 50_000); + turn.usage.requests = Some(2); + let model_call = |minute: i64, input: u64, read: u64, model: &str| ModelCall { + completed_at: at(1, 9, 0) + chrono::Duration::minutes(minute), + requested_at: None, + input_tokens: Some(input), + cache_read_tokens: Some(read), + cache_creation_tokens: None, + after_compaction: false, + model: Some(model.to_string()), + }; + turn.context = Some(CallContext { + calls: vec![ + model_call(0, 10, 50_000, "unpriced-model"), + model_call(1, 50_000, 0, "unpriced-model"), + ], + ..CallContext::default() + }); + store.insert_events(&[turn]).expect("insert"); + let diagnostics = store + .cache_report(&query, &Utc) + .expect("report") + .diagnostics; + assert_eq!( + ( + diagnostics.losses, + diagnostics.unpriced_losses, + diagnostics.missed_cost_micro_usd + ), + (1, 1, 0) + ); +} diff --git a/crates/statsai-store/src/tests/events.rs b/crates/statsai-store/src/tests/events.rs index eb74fc9..0fe92f0 100644 --- a/crates/statsai-store/src/tests/events.rs +++ b/crates/statsai-store/src/tests/events.rs @@ -68,6 +68,7 @@ fn inserts_events_idempotently() { }, created_at: now, imported_at: now, + context: None, }; assert!(store.insert_event(&event).expect("insert")); diff --git a/crates/statsai-store/src/tests/mod.rs b/crates/statsai-store/src/tests/mod.rs index d1bf1af..4a4c1f5 100644 --- a/crates/statsai-store/src/tests/mod.rs +++ b/crates/statsai-store/src/tests/mod.rs @@ -1,6 +1,7 @@ pub(crate) use super::*; mod accounts; +mod cache; mod dedupe; mod events; mod open; diff --git a/crates/statsai-store/src/tests/rollups/build.rs b/crates/statsai-store/src/tests/rollups/build.rs index 8e58c2d..bdb893d 100644 --- a/crates/statsai-store/src/tests/rollups/build.rs +++ b/crates/statsai-store/src/tests/rollups/build.rs @@ -30,7 +30,7 @@ fn sync_rollup_counts_underlying_requests_in_codex_turn_events() { ..ModelInfo::default() }); - let summary = build_sync_rollup_summary(&[first, second, legacy]); + let summary = build_sync_rollup_summary(&[first, second, legacy], None); assert_eq!(summary.usage.requests, Some(9)); assert_eq!(summary.models.len(), 2); @@ -188,7 +188,7 @@ fn sync_rollup_sums_micro_usd_before_rounding_to_cents() { event.cost.set_estimated_micro_usd(2_250); let events = vec![event; 1_000]; - let summary = build_sync_rollup_summary(&events); + let summary = build_sync_rollup_summary(&events, None); assert_eq!( summary.cost.estimated_api_equivalent_micro_usd, diff --git a/crates/statsai-store/src/tests/support.rs b/crates/statsai-store/src/tests/support.rs index 0f6fc58..9baee87 100644 --- a/crates/statsai-store/src/tests/support.rs +++ b/crates/statsai-store/src/tests/support.rs @@ -77,6 +77,7 @@ pub(super) fn test_store_event( }, created_at: now, imported_at: now, + context: None, } } diff --git a/crates/statsai-store/src/tests/sync_state/pending.rs b/crates/statsai-store/src/tests/sync_state/pending.rs index 9bc0337..2be2a41 100644 --- a/crates/statsai-store/src/tests/sync_state/pending.rs +++ b/crates/statsai-store/src/tests/sync_state/pending.rs @@ -120,6 +120,9 @@ fn pending_http_sync_summary_counts_include_retirement_only_reconciliation() { ); store.upsert_source(&source).expect("source"); let target = "https://api.example.com/api/sync/batches"; + store + .record_sync_target_cache_health_support("http", target, true) + .expect("receiver support"); let event = test_store_event(&source, Utc::now(), "retired-event"); store.insert_event(&event).expect("event"); let rollups = store.all_sync_rollup_summaries().expect("initial rollups"); diff --git a/crates/statsai-store/tests/cache_health_fixture.rs b/crates/statsai-store/tests/cache_health_fixture.rs new file mode 100644 index 0000000..03f4c81 --- /dev/null +++ b/crates/statsai-store/tests/cache_health_fixture.rs @@ -0,0 +1,304 @@ +//! The shared prompt-cache fixture. +//! +//! Synthetic Claude Code, Codex, and Cursor usage goes through the store; the +//! daily summaries it syncs and the totals `statsai report cache` prints are +//! written to `fixtures/cache_health.json`. Receivers of the summaries test +//! against a copy of that file, so the CLI report, the synced daily objects, +//! and anything aggregating them agree on one set of numbers. +//! +//! Set `STATSAI_UPDATE_FIXTURES=1` to rewrite the file after an intended change. + +use chrono::{DateTime, TimeZone, Utc}; +use serde_json::json; +use statsai_core::{ + sanitize_summary_for_sync, CallContext, Confidence, CostInfo, EventId, EventSource, + IdentitySource, LocationOrigin, ModelCall, ModelInfo, ParseEvidence, PrivacyInfo, PrivacyMode, + SessionInfo, SourceId, SourceKind, UsageCounts, UsageEvent, USAGE_EVENT_SCHEMA_VERSION, +}; +use statsai_store::{CacheReportQuery, Store}; +use std::path::Path; + +const FIXTURE: &str = "tests/fixtures/cache_health.json"; + +fn at(day: u32, hour: u32, minute: u32, second: u32) -> DateTime { + Utc.with_ymd_and_hms(2026, 7, day, hour, minute, second) + .single() + .expect("time") +} + +struct Call { + provider: &'static str, + session: &'static str, + file: &'static str, + line: u64, + at: DateTime, + input: u64, + read: u64, + write: Option, +} + +impl Call { + fn event(&self) -> UsageEvent { + let id = format!("{}_{}_{:03}", self.provider, self.session, self.line); + UsageEvent { + schema_version: USAGE_EVENT_SCHEMA_VERSION.to_string(), + event_id: EventId(format!("event_{id}")), + device_id: "device_fixture".to_string(), + provider: self.provider.to_string(), + source_id: SourceId(format!("src_fixture_{}", self.provider)), + provider_account_id: None, + subscription_id: None, + source: EventSource { + adapter_id: "fixture".to_string(), + adapter_version: "0".to_string(), + source_kind: SourceKind::LocalAdapter, + location_origin: Some(LocationOrigin::Configured), + source_type: "jsonl".to_string(), + source_path_hash: None, + source_record_id: None, + parse_confidence: Confidence::High, + }, + session: SessionInfo { + session_id: self.session.to_string(), + local_session_id_hash: None, + title: None, + started_at: self.at, + ended_at: None, + duration_seconds: None, + turn_started_at: None, + }, + model: Some(ModelInfo { + // Priced models, so missed reuse has a cost to check. + normalized_name: Some( + match self.provider { + "codex" => "gpt-6-sol", + "claude_code" => "claude-sonnet-4-5", + _ => "cursor-model", + } + .to_string(), + ), + ..ModelInfo::default() + }), + usage: UsageCounts { + input_tokens: Some(self.input), + output_tokens: Some(100), + cache_creation_tokens: self.write, + cache_read_tokens: Some(self.read), + requests: Some(1), + ..UsageCounts::default() + }, + runtime: None, + cost: CostInfo { + currency: "USD".to_string(), + estimated_api_equivalent_usd: None, + provider_reported_usd: None, + estimated_api_equivalent_micro_usd: None, + provider_reported_micro_usd: None, + pricing_source: None, + pricing_version: None, + confidence: Confidence::Low, + }, + parse_evidence: Some(ParseEvidence { + event_key_version: "fixture".to_string(), + source_file_path_hash: Some(self.file.to_string()), + source_line_number: Some(self.line), + source_record_id: None, + model_inferred: false, + timestamp_inferred: false, + account_identity_source: IdentitySource::Unresolved, + }), + project: None, + git: None, + privacy: PrivacyInfo { + mode: PrivacyMode::MetadataOnly, + contains_prompt_text: false, + contains_response_text: false, + contains_file_paths: false, + }, + created_at: self.at, + imported_at: self.at, + context: None, + } + } +} + +fn claude(line: u64, at: DateTime, input: u64, read: u64, write: u64) -> UsageEvent { + Call { + provider: "claude_code", + session: "session_fixture_claude", + file: "claude-main", + line, + at, + input, + read, + write: Some(write), + } + .event() +} + +fn events() -> Vec { + let mut events = vec![ + // Day one: a cold start, growth, a short-gap partial loss, a pause. + claude(1, at(1, 9, 0, 0), 20, 0, 40_000), + claude(2, at(1, 9, 0, 30), 20, 40_000, 2_000), + claude(3, at(1, 9, 1, 10), 20, 15_000, 27_500), + claude(4, at(1, 9, 3, 0), 20, 42_500, 1_000), + claude(5, at(1, 9, 9, 0), 20, 43_500, 900), + // Across midnight after a long pause: a full loss. + claude(6, at(1, 23, 50, 0), 20, 44_400, 600), + claude(7, at(2, 1, 0, 0), 20, 0, 45_200), + claude(8, at(2, 1, 0, 40), 20, 45_200, 800), + // A request a router served without the cache: neither read nor written. + claude(11, at(2, 1, 2, 0), 46_020, 0, 0), + ]; + // A compaction rewrites the prompt: a restart, not a loss. + let mut compacted = claude(9, at(2, 1, 5, 0), 20, 0, 9_000); + compacted.context = Some(CallContext { + after_compaction: true, + ..CallContext::default() + }); + events.push(compacted); + // A sub-agent in the same session is its own stream. + let mut agent = claude(10, at(1, 9, 2, 0), 20, 0, 8_000); + agent.context = Some(CallContext { + agent_hash: Some("fixture_agent".to_string()), + ..CallContext::default() + }); + events.push(agent); + + // A Codex turn listing its calls, with recorded request starts. + let mut turn = Call { + provider: "codex", + session: "session_fixture_codex", + file: "codex-rollout", + line: 1, + at: at(2, 10, 0, 0), + input: 0, + read: 0, + write: Some(0), + } + .event(); + let calls = [ + (at(2, 10, 0, 10), at(2, 10, 0, 0), 30_000, 0), + (at(2, 10, 0, 30), at(2, 10, 0, 12), 3_000, 29_952), + (at(2, 10, 7, 0), at(2, 10, 6, 40), 26_000, 7_168), + ]; + turn.usage.requests = Some(calls.len() as u64); + turn.usage.input_tokens = Some(calls.iter().map(|call| call.2).sum()); + turn.usage.cache_read_tokens = Some(calls.iter().map(|call| call.3).sum()); + turn.created_at = at(2, 10, 7, 30); + turn.context = Some(CallContext { + calls: calls + .iter() + .map(|(completed_at, requested_at, input, read)| ModelCall { + completed_at: *completed_at, + requested_at: Some(*requested_at), + input_tokens: Some(*input), + cache_read_tokens: Some(*read), + cache_creation_tokens: Some(0), + after_compaction: false, + model: None, + }) + .collect(), + ..CallContext::default() + }); + events.push(turn); + // An older Codex turn without a per-call breakdown and without write + // telemetry: counted, not classified. + let mut legacy = Call { + provider: "codex", + session: "session_fixture_codex", + file: "codex-rollout", + line: 2, + at: at(2, 11, 0, 0), + input: 4_000, + read: 60_000, + write: None, + } + .event(); + legacy.usage.requests = Some(2); + events.push(legacy); + + // Cursor has no call order: token totals only. + events.push( + Call { + provider: "cursor", + session: "session_fixture_cursor", + file: "cursor-export", + line: 1, + at: at(2, 12, 0, 0), + input: 1_000, + read: 9_000, + write: Some(500), + } + .event(), + ); + events +} + +fn fixture() -> serde_json::Value { + let store = Store::in_memory().expect("store"); + store.insert_events(&events()).expect("insert"); + let mut summaries = store + .all_sync_rollup_summaries() + .expect("summaries") + .into_iter() + .map(sanitize_summary_for_sync) + .collect::>(); + summaries.sort_by(|left, right| left.summary_id.0.cmp(&right.summary_id.0)); + let report = store + .cache_report( + &CacheReportQuery::for_range(Some("2026-07-01"), Some("2026-07-02"), at(3, 0, 0, 0)) + .expect("query"), + &Utc, + ) + .expect("report"); + json!({ + "summaries": summaries, + "report": { + "totals": report.totals, + "diagnostics": report.diagnostics, + "gap_histogram": report.gap_histogram, + "days": report.days.iter().map(|day| json!({ + "day": day.day, + "losses": day.diagnostics.as_ref().map_or(0, |health| health.losses), + "missed": day.diagnostics.as_ref().map_or(0, |health| health.missed_tokens), + "missed_cost_micro_usd": day + .diagnostics + .as_ref() + .map_or(0, |health| health.missed_cost_micro_usd), + })).collect::>(), + }, + }) +} + +#[test] +fn the_shared_cache_health_fixture_is_current() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join(FIXTURE); + let current = serde_json::to_string_pretty(&fixture()).expect("json") + "\n"; + if std::env::var_os("STATSAI_UPDATE_FIXTURES").is_some() { + std::fs::write(&path, ¤t).expect("write fixture"); + return; + } + let stored = std::fs::read_to_string(&path).unwrap_or_default(); + assert!( + stored == current, + "{FIXTURE} is out of date; rerun with STATSAI_UPDATE_FIXTURES=1 and copy it to its receivers" + ); +} + +#[test] +fn the_fixture_report_matches_its_daily_summaries() { + let fixture = fixture(); + let mut merged = statsai_core::CacheHealthV1::default(); + for summary in fixture["summaries"].as_array().expect("summaries") { + if let Some(health) = summary.pointer("/metrics/cache_health") { + merged.merge(&serde_json::from_value(health.clone()).expect("cache health")); + } + } + assert_eq!( + serde_json::to_value(&merged).expect("json"), + fixture["report"]["diagnostics"] + ); + assert_eq!((merged.unclassifiable, merged.boundaries), (2, 1)); +} diff --git a/crates/statsai-store/tests/fixtures/cache_health.json b/crates/statsai-store/tests/fixtures/cache_health.json new file mode 100644 index 0000000..70047ba --- /dev/null +++ b/crates/statsai-store/tests/fixtures/cache_health.json @@ -0,0 +1,785 @@ +{ + "report": { + "days": [ + { + "day": "2026-07-01", + "losses": 1, + "missed": 27000, + "missed_cost_micro_usd": 93150 + }, + { + "day": "2026-07-02", + "losses": 2, + "missed": 67784, + "missed_cost_micro_usd": 196262 + } + ], + "diagnostics": { + "below_threshold": 1, + "boundaries": 1, + "comparable": 8, + "first_cold": 3, + "first_observed": 3, + "full_losses": 1, + "gap_comparable": [ + 3, + 1, + 2, + 0, + 0, + 0, + 0, + 2 + ], + "gap_losses": [ + 1, + 0, + 1, + 0, + 0, + 0, + 0, + 1 + ], + "gap_missed_tokens": [ + 27000, + 0, + 22784, + 0, + 0, + 0, + 0, + 45000 + ], + "losses": 3, + "missed_cost_micro_usd": 289412, + "missed_tokens": 94784, + "requests": 16, + "timing_estimated": 7, + "uncached": 1, + "unclassifiable": 2, + "version": 1, + "write_split_unavailable_input_tokens": 4000 + }, + "gap_histogram": [ + { + "comparable": 3, + "label": "<1m", + "losses": 1, + "max_minutes": 1, + "min_minutes": 0, + "missed_tokens": 27000 + }, + { + "comparable": 1, + "label": "1–5m", + "losses": 0, + "max_minutes": 5, + "min_minutes": 1, + "missed_tokens": 0 + }, + { + "comparable": 2, + "label": "5–10m", + "losses": 1, + "max_minutes": 10, + "min_minutes": 5, + "missed_tokens": 22784 + }, + { + "comparable": 0, + "label": "10–15m", + "losses": 0, + "max_minutes": 15, + "min_minutes": 10, + "missed_tokens": 0 + }, + { + "comparable": 0, + "label": "15–30m", + "losses": 0, + "max_minutes": 30, + "min_minutes": 15, + "missed_tokens": 0 + }, + { + "comparable": 0, + "label": "30–45m", + "losses": 0, + "max_minutes": 45, + "min_minutes": 30, + "missed_tokens": 0 + }, + { + "comparable": 0, + "label": "45–60m", + "losses": 0, + "max_minutes": 60, + "min_minutes": 45, + "missed_tokens": 0 + }, + { + "comparable": 2, + "label": "60m+", + "losses": 1, + "max_minutes": null, + "min_minutes": 60, + "missed_tokens": 45000 + } + ], + "totals": { + "avg_logical_input_per_request": 34261.17647058824, + "cache_creation_tokens": 135500, + "cache_hit_ratio": 0.578119634640478, + "cache_read_tokens": 336720, + "input_tokens": 110220, + "logical_input_tokens": 582440, + "requests": 17 + } + }, + "summaries": [ + { + "cost": { + "confidence": "medium", + "currency": "USD", + "estimated_api_equivalent_usd": null, + "pricing_source": "local_rollup", + "pricing_version": null, + "provider_reported_usd": null + }, + "device_id": "device_fixture", + "imported_at": "2026-07-02T11:00:00Z", + "metadata": { + "last_computed_at": "2026-07-02T11:00:00Z", + "summary_format": "daily_rollup.v1", + "summary_version": "14", + "total_messages": null, + "total_sessions": 1 + }, + "metrics": { + "active_seconds": null, + "assistant_messages": null, + "cache_health": { + "below_threshold": 1, + "boundaries": 0, + "comparable": 1, + "first_cold": 1, + "first_observed": 1, + "full_losses": 0, + "gap_comparable": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "gap_losses": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "gap_missed_tokens": [ + 0, + 0, + 22784, + 0, + 0, + 0, + 0, + 0 + ], + "losses": 1, + "missed_cost_micro_usd": 41012, + "missed_tokens": 22784, + "requests": 5, + "timing_estimated": 0, + "unclassifiable": 2, + "version": 1, + "write_split_unavailable_input_tokens": 4000 + }, + "cache_hit_ratio": { + "avg": 0.6618419683728672, + "max": 0.9375, + "min": 0.3861839367457345, + "p50": 0.3861839367457345, + "p95": 0.9375, + "samples": 2, + "sum": 1.3236839367457345 + }, + "developer_messages": null, + "generated_tps": null, + "latency_ms": null, + "overall_generated_tps": null, + "overall_visible_tps": null, + "reasoning_share": { + "avg": 0.0, + "max": 0.0, + "min": 0.0, + "p50": 0.0, + "p95": 0.0, + "samples": 2, + "sum": 0.0 + }, + "time_to_first_token_ms": null, + "total_messages": null, + "tracked_output_tokens": null, + "tracked_reasoning_tokens": null, + "tracked_requests": null, + "user_messages": null, + "visible_tps": null + }, + "model": null, + "models": [ + { + "cost": { + "confidence": "medium", + "currency": "USD", + "estimated_api_equivalent_usd": null, + "pricing_source": "local_rollup", + "pricing_version": null, + "provider_reported_usd": null + }, + "model": { + "name": null, + "normalized_name": "gpt-6-sol", + "provider_model_id": null, + "reasoning_level": null, + "reasoning_level_raw": null, + "speed": null + }, + "usage": { + "cache_creation_1h_tokens": 0, + "cache_creation_5m_tokens": 0, + "cache_creation_tokens": 0, + "cache_read_tokens": 97120, + "input_tokens": 63000, + "local_eval_tokens": null, + "local_prompt_eval_tokens": null, + "output_tokens": 200, + "reasoning_tokens": 0, + "requests": 5, + "total_tokens": 160320 + } + } + ], + "observed_at": "2026-07-02T11:00:00Z", + "parse_evidence": null, + "period_end": "2026-07-03T00:00:00Z", + "period_start": "2026-07-02T00:00:00Z", + "privacy": { + "contains_file_paths": false, + "contains_prompt_text": false, + "contains_response_text": false, + "mode": "metadata_only" + }, + "project": null, + "provider": "codex", + "provider_account_id": null, + "schema_version": "usage_summary.v1", + "source": { + "adapter_id": "fixture", + "adapter_version": "0", + "location_origin": "configured", + "parse_confidence": "high", + "source_kind": "local_adapter", + "source_path_hash": null, + "source_record_id": null, + "source_type": "jsonl" + }, + "source_id": "src_fixture_codex", + "summary_id": "sum_547c6aa40f5895680f0045854f4a95e3", + "usage": { + "cache_creation_1h_tokens": 0, + "cache_creation_5m_tokens": 0, + "cache_creation_tokens": 0, + "cache_read_tokens": 97120, + "input_tokens": 63000, + "local_eval_tokens": null, + "local_prompt_eval_tokens": null, + "output_tokens": 200, + "reasoning_tokens": 0, + "requests": 5, + "total_tokens": 160320 + } + }, + { + "cost": { + "confidence": "medium", + "currency": "USD", + "estimated_api_equivalent_usd": null, + "pricing_source": "local_rollup", + "pricing_version": null, + "provider_reported_usd": null + }, + "device_id": "device_fixture", + "imported_at": "2026-07-02T12:00:00Z", + "metadata": { + "last_computed_at": "2026-07-02T12:00:00Z", + "summary_format": "daily_rollup.v1", + "summary_version": "14", + "total_messages": null, + "total_sessions": 1 + }, + "metrics": { + "active_seconds": null, + "assistant_messages": null, + "cache_hit_ratio": { + "avg": 0.8571428571428571, + "max": 0.8571428571428571, + "min": 0.8571428571428571, + "p50": 0.8571428571428571, + "p95": 0.8571428571428571, + "samples": 1, + "sum": 0.8571428571428571 + }, + "developer_messages": null, + "generated_tps": null, + "latency_ms": null, + "overall_generated_tps": null, + "overall_visible_tps": null, + "reasoning_share": { + "avg": 0.0, + "max": 0.0, + "min": 0.0, + "p50": 0.0, + "p95": 0.0, + "samples": 1, + "sum": 0.0 + }, + "time_to_first_token_ms": null, + "total_messages": null, + "tracked_output_tokens": null, + "tracked_reasoning_tokens": null, + "tracked_requests": null, + "user_messages": null, + "visible_tps": null + }, + "model": null, + "models": [ + { + "cost": { + "confidence": "medium", + "currency": "USD", + "estimated_api_equivalent_usd": null, + "pricing_source": "local_rollup", + "pricing_version": null, + "provider_reported_usd": null + }, + "model": { + "name": null, + "normalized_name": "cursor-model", + "provider_model_id": null, + "reasoning_level": null, + "reasoning_level_raw": null, + "speed": null + }, + "usage": { + "cache_creation_1h_tokens": 0, + "cache_creation_5m_tokens": 0, + "cache_creation_tokens": 500, + "cache_read_tokens": 9000, + "input_tokens": 1000, + "local_eval_tokens": null, + "local_prompt_eval_tokens": null, + "output_tokens": 100, + "reasoning_tokens": 0, + "requests": 1, + "total_tokens": 10600 + } + } + ], + "observed_at": "2026-07-02T12:00:00Z", + "parse_evidence": null, + "period_end": "2026-07-03T00:00:00Z", + "period_start": "2026-07-02T00:00:00Z", + "privacy": { + "contains_file_paths": false, + "contains_prompt_text": false, + "contains_response_text": false, + "mode": "metadata_only" + }, + "project": null, + "provider": "cursor", + "provider_account_id": null, + "schema_version": "usage_summary.v1", + "source": { + "adapter_id": "fixture", + "adapter_version": "0", + "location_origin": "configured", + "parse_confidence": "high", + "source_kind": "local_adapter", + "source_path_hash": null, + "source_record_id": null, + "source_type": "jsonl" + }, + "source_id": "src_fixture_cursor", + "summary_id": "sum_8e4eef2e63c1cf9e4c9c0caa09b6f554", + "usage": { + "cache_creation_1h_tokens": 0, + "cache_creation_5m_tokens": 0, + "cache_creation_tokens": 500, + "cache_read_tokens": 9000, + "input_tokens": 1000, + "local_eval_tokens": null, + "local_prompt_eval_tokens": null, + "output_tokens": 100, + "reasoning_tokens": 0, + "requests": 1, + "total_tokens": 10600 + } + }, + { + "cost": { + "confidence": "medium", + "currency": "USD", + "estimated_api_equivalent_usd": null, + "pricing_source": "local_rollup", + "pricing_version": null, + "provider_reported_usd": null + }, + "device_id": "device_fixture", + "imported_at": "2026-07-02T01:05:00Z", + "metadata": { + "last_computed_at": "2026-07-02T01:05:00Z", + "summary_format": "daily_rollup.v1", + "summary_version": "14", + "total_messages": null, + "total_sessions": 1 + }, + "metrics": { + "active_seconds": null, + "assistant_messages": null, + "cache_health": { + "below_threshold": 0, + "boundaries": 1, + "comparable": 2, + "first_cold": 0, + "first_observed": 0, + "full_losses": 1, + "gap_comparable": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1 + ], + "gap_losses": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1 + ], + "gap_missed_tokens": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 45000 + ], + "losses": 1, + "missed_cost_micro_usd": 155250, + "missed_tokens": 45000, + "requests": 4, + "timing_estimated": 2, + "uncached": 1, + "unclassifiable": 0, + "version": 1, + "write_split_unavailable_input_tokens": 0 + }, + "cache_hit_ratio": { + "avg": 0.24554541503694047, + "max": 0.982181660147762, + "min": 0.0, + "p50": 0.0, + "p95": 0.982181660147762, + "samples": 4, + "sum": 0.982181660147762 + }, + "developer_messages": null, + "generated_tps": null, + "latency_ms": null, + "overall_generated_tps": null, + "overall_visible_tps": null, + "reasoning_share": { + "avg": 0.0, + "max": 0.0, + "min": 0.0, + "p50": 0.0, + "p95": 0.0, + "samples": 4, + "sum": 0.0 + }, + "time_to_first_token_ms": null, + "total_messages": null, + "tracked_output_tokens": null, + "tracked_reasoning_tokens": null, + "tracked_requests": null, + "user_messages": null, + "visible_tps": null + }, + "model": null, + "models": [ + { + "cost": { + "confidence": "medium", + "currency": "USD", + "estimated_api_equivalent_usd": null, + "pricing_source": "local_rollup", + "pricing_version": null, + "provider_reported_usd": null + }, + "model": { + "name": null, + "normalized_name": "claude-sonnet-4-5", + "provider_model_id": null, + "reasoning_level": null, + "reasoning_level_raw": null, + "speed": null + }, + "usage": { + "cache_creation_1h_tokens": 0, + "cache_creation_5m_tokens": 0, + "cache_creation_tokens": 55000, + "cache_read_tokens": 45200, + "input_tokens": 46080, + "local_eval_tokens": null, + "local_prompt_eval_tokens": null, + "output_tokens": 400, + "reasoning_tokens": 0, + "requests": 4, + "total_tokens": 146680 + } + } + ], + "observed_at": "2026-07-02T01:05:00Z", + "parse_evidence": null, + "period_end": "2026-07-03T00:00:00Z", + "period_start": "2026-07-02T00:00:00Z", + "privacy": { + "contains_file_paths": false, + "contains_prompt_text": false, + "contains_response_text": false, + "mode": "metadata_only" + }, + "project": null, + "provider": "claude_code", + "provider_account_id": null, + "schema_version": "usage_summary.v1", + "source": { + "adapter_id": "fixture", + "adapter_version": "0", + "location_origin": "configured", + "parse_confidence": "high", + "source_kind": "local_adapter", + "source_path_hash": null, + "source_record_id": null, + "source_type": "jsonl" + }, + "source_id": "src_fixture_claude_code", + "summary_id": "sum_9ebe14eb7cbd9f700433ec92c14e824d", + "usage": { + "cache_creation_1h_tokens": 0, + "cache_creation_5m_tokens": 0, + "cache_creation_tokens": 55000, + "cache_read_tokens": 45200, + "input_tokens": 46080, + "local_eval_tokens": null, + "local_prompt_eval_tokens": null, + "output_tokens": 400, + "reasoning_tokens": 0, + "requests": 4, + "total_tokens": 146680 + } + }, + { + "cost": { + "confidence": "medium", + "currency": "USD", + "estimated_api_equivalent_usd": null, + "pricing_source": "local_rollup", + "pricing_version": null, + "provider_reported_usd": null + }, + "device_id": "device_fixture", + "imported_at": "2026-07-01T23:50:00Z", + "metadata": { + "last_computed_at": "2026-07-01T23:50:00Z", + "summary_format": "daily_rollup.v1", + "summary_version": "14", + "total_messages": null, + "total_sessions": 1 + }, + "metrics": { + "active_seconds": null, + "assistant_messages": null, + "cache_health": { + "below_threshold": 0, + "boundaries": 0, + "comparable": 5, + "first_cold": 2, + "first_observed": 2, + "full_losses": 0, + "gap_comparable": [ + 2, + 1, + 1, + 0, + 0, + 0, + 0, + 1 + ], + "gap_losses": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "gap_missed_tokens": [ + 27000, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "losses": 1, + "missed_cost_micro_usd": 93150, + "missed_tokens": 27000, + "requests": 7, + "timing_estimated": 5, + "unclassifiable": 0, + "version": 1, + "write_split_unavailable_input_tokens": 0 + }, + "cache_hit_ratio": { + "avg": 0.6066831814028905, + "max": 0.986228342958685, + "min": 0.0, + "p50": 0.951927653498334, + "p95": 0.986228342958685, + "samples": 7, + "sum": 4.246782269820233 + }, + "developer_messages": null, + "generated_tps": null, + "latency_ms": null, + "overall_generated_tps": null, + "overall_visible_tps": null, + "reasoning_share": { + "avg": 0.0, + "max": 0.0, + "min": 0.0, + "p50": 0.0, + "p95": 0.0, + "samples": 7, + "sum": 0.0 + }, + "time_to_first_token_ms": null, + "total_messages": null, + "tracked_output_tokens": null, + "tracked_reasoning_tokens": null, + "tracked_requests": null, + "user_messages": null, + "visible_tps": null + }, + "model": null, + "models": [ + { + "cost": { + "confidence": "medium", + "currency": "USD", + "estimated_api_equivalent_usd": null, + "pricing_source": "local_rollup", + "pricing_version": null, + "provider_reported_usd": null + }, + "model": { + "name": null, + "normalized_name": "claude-sonnet-4-5", + "provider_model_id": null, + "reasoning_level": null, + "reasoning_level_raw": null, + "speed": null + }, + "usage": { + "cache_creation_1h_tokens": 0, + "cache_creation_5m_tokens": 0, + "cache_creation_tokens": 80000, + "cache_read_tokens": 185400, + "input_tokens": 140, + "local_eval_tokens": null, + "local_prompt_eval_tokens": null, + "output_tokens": 700, + "reasoning_tokens": 0, + "requests": 7, + "total_tokens": 266240 + } + } + ], + "observed_at": "2026-07-01T23:50:00Z", + "parse_evidence": null, + "period_end": "2026-07-02T00:00:00Z", + "period_start": "2026-07-01T00:00:00Z", + "privacy": { + "contains_file_paths": false, + "contains_prompt_text": false, + "contains_response_text": false, + "mode": "metadata_only" + }, + "project": null, + "provider": "claude_code", + "provider_account_id": null, + "schema_version": "usage_summary.v1", + "source": { + "adapter_id": "fixture", + "adapter_version": "0", + "location_origin": "configured", + "parse_confidence": "high", + "source_kind": "local_adapter", + "source_path_hash": null, + "source_record_id": null, + "source_type": "jsonl" + }, + "source_id": "src_fixture_claude_code", + "summary_id": "sum_bf315597fd8d01330eb4d20bcff5b1ab", + "usage": { + "cache_creation_1h_tokens": 0, + "cache_creation_5m_tokens": 0, + "cache_creation_tokens": 80000, + "cache_read_tokens": 185400, + "input_tokens": 140, + "local_eval_tokens": null, + "local_prompt_eval_tokens": null, + "output_tokens": 700, + "reasoning_tokens": 0, + "requests": 7, + "total_tokens": 266240 + } + } + ] +} diff --git a/crates/statsai-sync/src/lib.rs b/crates/statsai-sync/src/lib.rs index 6fd5a92..3e54589 100644 --- a/crates/statsai-sync/src/lib.rs +++ b/crates/statsai-sync/src/lib.rs @@ -857,6 +857,7 @@ mod tests { }, created_at: Utc::now(), imported_at: Utc::now(), + context: None, } } diff --git a/crates/statsai/src/cli/args/mod.rs b/crates/statsai/src/cli/args/mod.rs index 0d80179..b472a16 100644 --- a/crates/statsai/src/cli/args/mod.rs +++ b/crates/statsai/src/cli/args/mod.rs @@ -280,6 +280,8 @@ pub(crate) enum SchemaSubcommand { SyncBatch, #[command(about = "Print the quota_window_sync_projection.v1 JSON Schema")] QuotaWindowProjection, + #[command(about = "Print the cache_report.v1 JSON Schema")] + CacheReport, } #[derive(Debug, Args)] diff --git a/crates/statsai/src/cli/args/report.rs b/crates/statsai/src/cli/args/report.rs index 55adce2..8dae40f 100644 --- a/crates/statsai/src/cli/args/report.rs +++ b/crates/statsai/src/cli/args/report.rs @@ -56,4 +56,31 @@ pub(crate) enum ReportSubcommand { #[arg(long, help = "Include subscription-value rows")] subscriptions: bool, }, + #[command(about = "Show prompt-cache reuse, suspected cache losses, and context per call")] + Cache(CacheReportArgs), +} + +#[derive(Debug, Args)] +pub(crate) struct CacheReportArgs { + #[arg( + long, + help = "Range start (YYYY-MM-DD or RFC3339). Date-only values are UTC days. Defaults to the last 7 days" + )] + pub(crate) from: Option, + #[arg(long, help = "Range end (YYYY-MM-DD or RFC3339). Defaults to now")] + pub(crate) to: Option, + #[arg(long, conflicts_with_all = ["from", "to"], help = "Report all stored history")] + pub(crate) all: bool, + #[arg(long, help = "Provider filter")] + pub(crate) provider: Option, + #[arg(long, help = "Provider account id, or `unassigned`")] + pub(crate) account: Option, + #[arg(long, help = "Session id (session_…) or a prefix of one")] + pub(crate) session: Option, + #[arg(long, help = "Add local 10-minute buckets of calls")] + pub(crate) timeline: bool, + #[arg(long, help = "List every call with its verdict and evidence")] + pub(crate) details: bool, + #[arg(long, help = "Output as JSON")] + pub(crate) json: bool, } diff --git a/crates/statsai/src/cli/cache.rs b/crates/statsai/src/cli/cache.rs new file mode 100644 index 0000000..e409fa2 --- /dev/null +++ b/crates/statsai/src/cli/cache.rs @@ -0,0 +1,363 @@ +use anyhow::Result; +use chrono::{Local, Utc}; +use statsai_core::{ + micro_usd_to_cents_rounded, AnalyzedCacheCall, CacheBoundaryReason, CacheCallClass, + CacheGapTiming, CacheLoss, CacheReport, CacheTokenTotals, CacheUnclassifiableReason, + CacheWriteTtl, CACHE_LOSS_MIN_PERCENT, CACHE_LOSS_MIN_TOKENS, +}; +use statsai_store::{CacheReportQuery, Store}; + +use super::args::CacheReportArgs; +use super::format::{format_cost, format_u64, truncate_label}; +use super::source::canonical_provider; + +pub(crate) fn cache_report(args: CacheReportArgs, store: &Store) -> Result<()> { + let report = store.cache_report(&cache_report_query(&args, Utc::now())?, &Local)?; + if args.json { + println!("{}", serde_json::to_string_pretty(&report)?); + } else { + print_cache_report(&report); + } + Ok(()) +} + +pub(super) fn cache_report_query( + args: &CacheReportArgs, + now: chrono::DateTime, +) -> Result { + let mut query = if args.all { + CacheReportQuery::all_time(now) + } else { + CacheReportQuery::for_range(args.from.as_deref(), args.to.as_deref(), now)? + }; + query.provider = args + .provider + .as_deref() + .map(canonical_provider) + .transpose()?; + query.account = args.account.clone(); + query.session = args.session.clone(); + query.timeline = args.timeline; + query.details = args.details; + Ok(query) +} + +fn print_cache_report(report: &CacheReport) { + println!("statsai cache report: {}", report.label); + match report.since { + Some(since) => println!( + "range: {} to {}", + since.to_rfc3339(), + report.until.to_rfc3339() + ), + None => println!( + "range: all stored events through {}", + report.until.to_rfc3339() + ), + } + println!(); + print_totals(report); + println!(); + print_diagnostics(report); + if !report.providers.is_empty() { + println!(); + print_providers(report); + } + if let Some(timeline) = report.timeline.as_deref() { + println!(); + print_timeline(timeline); + } + if let Some(details) = report.details.as_deref() { + println!(); + print_details(details); + } +} + +fn print_totals(report: &CacheReport) { + let totals = &report.totals; + let unsplit = report.diagnostics.write_split_unavailable_input_tokens; + println!( + "{:<20}{} of {} logical input tokens", + "cache hit", + format_share(totals.cache_hit_ratio), + format_u64(totals.logical_input_tokens) + ); + let mut breakdown = format!( + "cache reads {} · cache writes {} · ordinary input {}", + format_u64(totals.cache_read_tokens), + format_u64(totals.cache_creation_tokens), + format_u64(totals.input_tokens.saturating_sub(unsplit)) + ); + if unsplit > 0 { + breakdown.push_str(&format!( + " · uncached input, writes not reported {}", + format_u64(unsplit) + )); + } + println!("{:<20}{breakdown}", "input breakdown"); + println!( + "{:<20}{} calls · {} input tokens per call", + "context processed", + format_u64(totals.requests), + format_average(totals) + ); +} + +fn format_average(totals: &CacheTokenTotals) -> String { + totals + .avg_logical_input_per_request + .map(|average| format_u64(average.round() as u64)) + .unwrap_or_else(|| "—".to_string()) +} + +fn print_diagnostics(report: &CacheReport) { + let health = &report.diagnostics; + println!("diagnostics (detector v{})", report.detector_version); + if health.requests == 0 { + println!(" no Claude Code or Codex calls in this range"); + return; + } + println!( + "{:<20}{} analyzed · {} unclassifiable · {} timing-estimated", + "coverage", + format_u64(health.analyzed()), + format_u64(health.unclassifiable), + format_u64(health.timing_estimated) + ); + println!( + "{:<20}{} streams · {} cold", + "first observed", + format_u64(health.first_observed), + format_u64(health.first_cold) + ); + let boundaries = &report.boundaries; + println!( + "{:<20}{} (model change {} · compaction {} · evidence gap {})", + "restarts", + format_u64(health.boundaries), + format_u64(boundaries.model_change), + format_u64(boundaries.compaction), + format_u64(boundaries.evidence_gap) + ); + println!( + "{:<20}{} · {} suspected losses ({} full · {} partial)", + "comparable calls", + format_u64(health.comparable), + format_u64(health.losses), + format_u64(health.full_losses), + format_u64(health.losses.saturating_sub(health.full_losses)) + ); + if health.losses > 0 { + let logical = report.totals.logical_input_tokens; + println!( + "{:<20}{} reusable tokens not read from the cache ({} of logical input)", + "missed reuse", + format_u64(health.missed_tokens), + format_share((logical > 0).then(|| health.missed_tokens as f64 / logical as f64)) + ); + let mut cost = format!( + "{} extra at API prices", + format_cost(Some(micro_usd_to_cents_rounded( + i64::try_from(health.missed_cost_micro_usd).unwrap_or(i64::MAX) + ))) + ); + if health.unpriced_losses > 0 { + cost.push_str(&format!( + ", leaving out {} losses on models without known pricing", + format_u64(health.unpriced_losses) + )); + } + println!("{:<20}{cost}", "missed reuse cost"); + } + if health.uncached > 0 { + println!( + "{:<20}{} calls neither read nor wrote the cache, so are not judged", + "cache not used", + format_u64(health.uncached) + ); + } + if health.below_threshold > 0 { + println!( + "{:<20}{} calls followed less than {} reusable tokens", + "too small to judge", + format_u64(health.below_threshold), + format_u64(CACHE_LOSS_MIN_TOKENS) + ); + } + println!(); + println!( + "{:<26} {:>12} {:>10} {:>10} {:>16}", + "gap since previous call", "comparable", "losses", "loss rate", "missed tokens" + ); + for row in &report.gap_histogram { + println!( + "{:<26} {:>12} {:>10} {:>10} {:>16}", + row.label, + format_u64(row.comparable), + format_u64(row.losses), + format_share((row.comparable > 0).then(|| row.losses as f64 / row.comparable as f64)), + format_u64(row.missed_tokens) + ); + } + println!(); + println!( + "A suspected loss is a call whose cached reads fell at least {}% and {} tokens below", + CACHE_LOSS_MIN_PERCENT, + format_u64(CACHE_LOSS_MIN_TOKENS) + ); + println!("the context the previous call left reusable. It is read from token counts, not"); + println!("confirmed by the provider. Timing-estimated gaps run between recorded responses"); + println!("and include the call's own run time."); +} + +fn print_providers(report: &CacheReport) { + println!( + "{:<14} {:>10} {:>8} {:>16} {:>12} {:>8}", + "provider", "calls", "hit", "input/call", "comparable", "losses" + ); + for row in &report.providers { + let health = row.diagnostics.as_ref(); + println!( + "{:<14} {:>10} {:>8} {:>16} {:>12} {:>8}", + truncate_label(&row.provider, 14), + format_u64(row.totals.requests), + format_share(row.totals.cache_hit_ratio), + format_average(&row.totals), + optional_count(health.map(|health| health.comparable)), + optional_count(health.map(|health| health.losses)), + ); + } +} + +fn print_timeline(timeline: &[statsai_core::CacheTimelineRow]) { + println!( + "{:<26} {:>8} {:>14} {:>8} {:>7} {:>9} {:>7}", + "local 10-minute bucket", "calls", "context", "hit", "first", "restarts", "losses" + ); + for row in timeline { + println!( + "{:<26} {:>8} {:>14} {:>8} {:>7} {:>9} {:>7}", + row.start, + format_u64(row.requests), + format_u64(row.logical_input_tokens), + format_share(row.cache_hit_ratio), + format_u64(row.first_observed), + format_u64(row.boundaries), + format_u64(row.losses) + ); + } +} + +fn print_details(details: &[AnalyzedCacheCall]) { + println!( + "{:<19} {:<12} {:<16} {:<18} {:>10} {:>10} {:>9} {:>10} {:>8} verdict", + "completed (local)", + "provider", + "session", + "model", + "input", + "read", + "write", + "baseline", + "gap" + ); + for call in details { + let agent = call + .agent_hash + .as_deref() + .map(|hash| format!("/{}", &hash[..hash.len().min(6)])) + .unwrap_or_default(); + println!( + "{:<19} {:<12} {:<16} {:<18} {:>10} {:>10} {:>9} {:>10} {:>8} {}", + call.completed_at + .with_timezone(&Local) + .format("%Y-%m-%d %H:%M:%S"), + truncate_label(&call.provider, 12), + truncate_label(&format!("{}{agent}", call.session_id), 16), + truncate_label(call.model.as_deref().unwrap_or("unknown"), 18), + format_u64(call.input_tokens), + format_u64(call.cache_read_tokens), + call.cache_creation_tokens + .map(format_u64) + .unwrap_or_else(|| "—".to_string()), + call.verdict + .baseline_tokens + .map(format_u64) + .unwrap_or_else(|| "—".to_string()), + format_gap(call), + verdict_label(call) + ); + } +} + +fn format_gap(call: &AnalyzedCacheCall) -> String { + let Some(seconds) = call.verdict.gap_seconds else { + return "—".to_string(); + }; + let text = if seconds >= 3600 { + format!("{}h{:02}m", seconds / 3600, (seconds % 3600) / 60) + } else if seconds >= 60 { + format!("{}m{:02}s", seconds / 60, seconds % 60) + } else { + format!("{seconds}s") + }; + match call.verdict.gap_timing { + Some(CacheGapTiming::RecordedEvents) => format!("~{text}"), + _ => text, + } +} + +fn verdict_label(call: &AnalyzedCacheCall) -> String { + let verdict = &call.verdict; + let mut label = match verdict.class { + CacheCallClass::FirstObserved if call.cache_read_tokens == 0 => { + "first observed, cold".to_string() + } + CacheCallClass::FirstObserved => "first observed".to_string(), + CacheCallClass::Boundary => match verdict.boundary { + Some(CacheBoundaryReason::ModelChange) => format!( + "restart: model changed from {}", + verdict.previous_model.as_deref().unwrap_or("unknown") + ), + Some(CacheBoundaryReason::Compaction) => "restart: after compaction".to_string(), + Some(CacheBoundaryReason::EvidenceGap) | None => { + "restart: after unclassifiable usage".to_string() + } + }, + CacheCallClass::BelowThreshold => "too small to judge".to_string(), + CacheCallClass::Comparable => match verdict.loss { + Some(CacheLoss::Full) => "suspected loss, nothing read".to_string(), + Some(CacheLoss::Partial) => "suspected loss, partial reuse".to_string(), + None => "reused".to_string(), + }, + CacheCallClass::Uncached => "cache not used".to_string(), + CacheCallClass::Unclassifiable => format!( + "unclassifiable: {}", + match verdict.unclassifiable { + Some(CacheUnclassifiableReason::TimestampMissing) => "no timestamp", + Some(CacheUnclassifiableReason::Aggregated) => "calls recorded as one total", + Some(CacheUnclassifiableReason::NoInputTelemetry) => "no input counts", + Some(CacheUnclassifiableReason::AmbiguousOrder) => "order not recorded", + None => "unknown", + } + ), + }; + if let Some(ttl) = verdict.previous_write_ttl { + label.push_str(match ttl { + CacheWriteTtl::FiveMinutes => " · previous writes 5m", + CacheWriteTtl::OneHour => " · previous writes 1h", + CacheWriteTtl::Mixed => " · previous writes 5m+1h", + }); + } + label +} + +fn format_share(value: Option) -> String { + value + .map(|value| format!("{:.1}%", value * 100.0)) + .unwrap_or_else(|| "—".to_string()) +} + +fn optional_count(value: Option) -> String { + value.map(format_u64).unwrap_or_else(|| "—".to_string()) +} diff --git a/crates/statsai/src/cli/mod.rs b/crates/statsai/src/cli/mod.rs index cecbb2e..d8012b1 100644 --- a/crates/statsai/src/cli/mod.rs +++ b/crates/statsai/src/cli/mod.rs @@ -2,6 +2,7 @@ pub(super) mod account; pub(super) mod activity; pub(super) mod args; pub(super) mod auth; +pub(super) mod cache; pub(super) mod conversation; pub(super) mod daemon; mod dispatch; diff --git a/crates/statsai/src/cli/report.rs b/crates/statsai/src/cli/report.rs index 1924db0..8559ae1 100644 --- a/crates/statsai/src/cli/report.rs +++ b/crates/statsai/src/cli/report.rs @@ -14,6 +14,9 @@ use super::format::{ }; pub(crate) fn report(command: ReportCommand, store: &Store) -> Result<()> { + if let ReportSubcommand::Cache(args) = command.command { + return super::cache::cache_report(args, store); + } let now = Utc::now(); let (report, json_output, verbose, include_subscriptions) = usage_report_from_command(command, store, now)?; @@ -60,6 +63,7 @@ pub(crate) fn usage_report_from_command( verbose, subscriptions, ), + ReportSubcommand::Cache(_) => bail!("the cache report has its own output"), }; let (since, until) = period.window(now); let (events, summaries) = match period { diff --git a/crates/statsai/src/cli/schema.rs b/crates/statsai/src/cli/schema.rs index b8cdc9a..563b4df 100644 --- a/crates/statsai/src/cli/schema.rs +++ b/crates/statsai/src/cli/schema.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use statsai_core::{QuotaWindowSyncProjectionV1, SyncBatch}; +use statsai_core::{CacheReport, QuotaWindowSyncProjectionV1, SyncBatch}; use super::args::{SchemaCommand, SchemaSubcommand}; @@ -13,6 +13,10 @@ pub(crate) fn schema(command: SchemaCommand) -> Result<()> { let schema = schemars::schema_for!(QuotaWindowSyncProjectionV1); println!("{}", serde_json::to_string_pretty(&schema)?); } + SchemaSubcommand::CacheReport => { + let schema = schemars::schema_for!(CacheReport); + println!("{}", serde_json::to_string_pretty(&schema)?); + } } Ok(()) } diff --git a/crates/statsai/src/cli/sync/batch.rs b/crates/statsai/src/cli/sync/batch.rs index 69367d2..42e75e5 100644 --- a/crates/statsai/src/cli/sync/batch.rs +++ b/crates/statsai/src/cli/sync/batch.rs @@ -14,7 +14,7 @@ use statsai_core::{ SyncAuthoritativeSnapshot, SyncBatch, TaskVerificationCursor, UsageEvent, UsageSummary, SYNC_BATCH_SCHEMA_VERSION, }; -use statsai_store::{QuotaQuery, Store}; +use statsai_store::{summary_for_sync_target, QuotaQuery, Store}; #[cfg(test)] use std::collections::BTreeMap; @@ -46,6 +46,13 @@ pub(crate) fn build_sync_batch_with_identity_key( let batch_id = format!("batch_{}", created_at.timestamp_millis()); let sync_preferences = effective_sync_preferences(store, command)?; let include_projects = sync_preferences.include_projects; + let accepts_cache_health = store.sync_target_accepts_cache_health(&command.sink, target)?; + let for_target = |summary: UsageSummary| { + summary_for_sync_target( + sanitize_summary_for_sync_with_projects(summary, include_projects), + accepts_cache_health, + ) + }; let payload_mode = sync_payload_mode(command)?; let state = if command.sink == "http" || command.since_last { store.sync_state(&command.sink, target)? @@ -81,7 +88,7 @@ pub(crate) fn build_sync_batch_with_identity_key( store .summaries()? .into_iter() - .map(|summary| sanitize_summary_for_sync_with_projects(summary, include_projects)) + .map(for_target) .filter(is_http_rollup_passthrough_summary) .collect() } else { @@ -93,7 +100,7 @@ pub(crate) fn build_sync_batch_with_identity_key( store .summaries_after(summary_cursor)? .into_iter() - .map(|summary| sanitize_summary_for_sync_with_projects(summary, include_projects)) + .map(for_target) .collect() }; let all_sources: Vec<_> = store @@ -323,7 +330,7 @@ pub(crate) fn build_sync_batch_with_identity_key( let all_rollups: Vec<_> = store .all_sync_rollup_summaries()? .into_iter() - .map(|summary| sanitize_summary_for_sync_with_projects(summary, include_projects)) + .map(for_target) .collect(); let current_authoritative_snapshot = SyncAuthoritativeSnapshot { snapshot_id: format!("{batch_id}_authoritative"), @@ -412,11 +419,7 @@ pub(crate) fn build_sync_batch_with_identity_key( } ); summaries.extend(passthrough_summaries); - summaries.extend( - rollups - .into_iter() - .map(|summary| sanitize_summary_for_sync_with_projects(summary, include_projects)), - ); + summaries.extend(rollups.into_iter().map(for_target)); } Ok(( @@ -624,6 +627,9 @@ pub(crate) fn sanitize_source_account_assignment_for_sync( } pub(crate) fn sanitize_event_for_sync(mut event: UsageEvent) -> UsageEvent { + // Per-call timing and sub-agent identity stay on the device. Daily + // summaries carry the counts derived from them. + event.context = None; event.source.source_record_id = None; if let Some(evidence) = event.parse_evidence.as_mut() { evidence.source_line_number = None; diff --git a/crates/statsai/src/cli/sync/http.rs b/crates/statsai/src/cli/sync/http.rs index a8dbccb..12d1675 100644 --- a/crates/statsai/src/cli/sync/http.rs +++ b/crates/statsai/src/cli/sync/http.rs @@ -634,6 +634,15 @@ pub(crate) fn remote_hosted_tasks_enabled(remote: &Value) -> bool { .unwrap_or(true) } +/// Whether the receiver stores `metrics.cache_health` in the version this +/// build writes. Receivers that predate it do not advertise it. +pub(crate) fn remote_accepts_cache_health(remote: Option<&Value>) -> bool { + remote + .and_then(|remote| remote.pointer("/capabilities/cacheHealth")) + .and_then(Value::as_u64) + == Some(u64::from(statsai_core::CACHE_HEALTH_VERSION)) +} + pub(crate) fn remote_code_change_identity_key(remote: &Value) -> Result> { let Some(value) = remote.pointer("/capabilities/codeChangeIdentityKey") else { return Ok(None); diff --git a/crates/statsai/src/cli/sync/mod.rs b/crates/statsai/src/cli/sync/mod.rs index d65e291..cc744cb 100644 --- a/crates/statsai/src/cli/sync/mod.rs +++ b/crates/statsai/src/cli/sync/mod.rs @@ -166,6 +166,11 @@ pub(crate) fn sync(command: SyncCommand, store: &Store, device_id: &str) -> Resu &target, preflight.remote.as_ref(), )?; + store.record_sync_target_cache_health_support( + &command.sink, + &target, + remote_accepts_cache_health(preflight.remote.as_ref()), + )?; Some(preflight) } else { None @@ -633,16 +638,23 @@ pub(crate) fn sync_local_verify( .map(|subscription| subscription.subscription_id.0.as_str()) .collect(), )?; + let accepts_cache_health = store.sync_target_accepts_cache_health(sink, target)?; + let for_target = |summary| { + statsai_store::summary_for_sync_target( + sanitize_summary_for_sync_with_projects(summary, include_projects), + accepts_cache_health, + ) + }; let passthrough_summaries: Vec<_> = store .summaries()? .into_iter() - .map(|summary| sanitize_summary_for_sync_with_projects(summary, include_projects)) + .map(for_target) .filter(is_http_rollup_passthrough_summary) .collect(); let rollup_summaries: Vec<_> = store .all_sync_rollup_summaries()? .into_iter() - .map(|summary| sanitize_summary_for_sync_with_projects(summary, include_projects)) + .map(for_target) .collect(); Ok(SyncLocalVerify { diff --git a/crates/statsai/src/cli/tests/args.rs b/crates/statsai/src/cli/tests/args.rs index 2a984ad..3d862ea 100644 --- a/crates/statsai/src/cli/tests/args.rs +++ b/crates/statsai/src/cli/tests/args.rs @@ -116,3 +116,27 @@ fn price_derived_commands_reprice_and_diagnostic_commands_do_not() { let doctor = Cli::try_parse_from(["statsai", "doctor"]).expect("parse doctor"); assert!(!command_reprices_persisted_usage(&doctor.command)); } + +#[test] +fn cache_report_all_covers_every_stored_call_and_excludes_a_range() { + let cli = Cli::try_parse_from(["statsai", "report", "cache", "--all", "--provider", "codex"]) + .expect("parse --all"); + let Command::Report(ReportCommand { + command: ReportSubcommand::Cache(args), + .. + }) = cli.command + else { + panic!("expected the cache report"); + }; + let now = Utc.with_ymd_and_hms(2026, 7, 10, 12, 0, 0).unwrap(); + let query = cache::cache_report_query(&args, now).expect("query"); + assert_eq!((query.since, query.until), (None, now)); + + for range in ["--from", "--to"] { + assert!( + Cli::try_parse_from(["statsai", "report", "cache", "--all", range, "2026-07-01"]) + .is_err(), + "--all with {range}" + ); + } +} diff --git a/crates/statsai/src/cli/tests/support.rs b/crates/statsai/src/cli/tests/support.rs index b119cdd..d148b9e 100644 --- a/crates/statsai/src/cli/tests/support.rs +++ b/crates/statsai/src/cli/tests/support.rs @@ -264,6 +264,7 @@ pub(super) fn test_event( }, created_at: started_at, imported_at: started_at, + context: None, } } diff --git a/crates/statsai/src/cli/tests/sync/batch.rs b/crates/statsai/src/cli/tests/sync/batch.rs index efe7405..0470ef5 100644 --- a/crates/statsai/src/cli/tests/sync/batch.rs +++ b/crates/statsai/src/cli/tests/sync/batch.rs @@ -2200,3 +2200,65 @@ fn excluding_projects_retires_hosted_sessions_and_excluding_sessions_keeps_them( .expect("sessions on"); assert_eq!(snapshot_session_ids(&command(false, true)), None); } + +#[test] +fn cache_health_reaches_only_receivers_that_advertise_it() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/codex-cache-health"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let mut event = test_event("codex", &source, Utc::now(), None, TokenParts::total(15)); + event.usage.requests = Some(1); + event.context = Some(statsai_core::CallContext { + requested_at: Some(event.created_at), + ..statsai_core::CallContext::default() + }); + store.insert_event(&event).expect("event"); + + let command = SyncCommand { + endpoint: Some("https://api.example.com/api/sync/batches".to_string()), + ..test_sync_command("http") + }; + let target = sync_target(&command).expect("target"); + let has_cache_health = |batch: &SyncBatch| { + batch.summaries.iter().any(|summary| { + summary + .metrics + .as_ref() + .is_some_and(|metrics| metrics.cache_health.is_some()) + }) + }; + + let (batch, _) = build_sync_batch(&command, &store, "device", &target).expect("batch"); + assert!(!has_cache_health(&batch), "unknown receivers get none"); + + store + .record_sync_target_cache_health_support("http", &target, true) + .expect("support"); + let (batch, _) = build_sync_batch(&command, &store, "device", &target).expect("batch"); + assert!(has_cache_health(&batch)); + + let (batch, _) = + build_sync_batch(&test_sync_command("stdout"), &store, "device", "stdout").expect("batch"); + assert!(batch.events.iter().all(|event| event.context.is_none())); +} + +#[test] +fn receivers_advertise_the_cache_health_version_they_store() { + let remote = |value: serde_json::Value| serde_json::json!({ "capabilities": value }); + assert!(remote_accepts_cache_health(Some(&remote( + serde_json::json!({ "cacheHealth": statsai_core::CACHE_HEALTH_VERSION }) + )))); + assert!(!remote_accepts_cache_health(Some(&remote( + serde_json::json!({ "cacheHealth": statsai_core::CACHE_HEALTH_VERSION + 1 }) + )))); + assert!(!remote_accepts_cache_health(Some(&remote( + serde_json::json!({ "hostedTasks": true }) + )))); + assert!(!remote_accepts_cache_health(None)); +} diff --git a/crates/statsai/src/cli/tests/sync/http.rs b/crates/statsai/src/cli/tests/sync/http.rs index 9a4dc6a..4adbbd6 100644 --- a/crates/statsai/src/cli/tests/sync/http.rs +++ b/crates/statsai/src/cli/tests/sync/http.rs @@ -332,14 +332,8 @@ fn http_rollup_sync_restarts_full_snapshot_after_snapshot_failure() { .expect("pending sources") .is_empty()); - let sync_rollups: Vec<_> = store - .all_sync_rollup_summaries() - .expect("rollups") - .into_iter() - .map(sanitize_summary_for_sync) - .collect(); let pending_rollups = store - .pending_summaries_for_sync("http", &target, &sync_rollups) + .pending_http_sync_rollup_summaries(&target) .expect("pending rollups"); assert!(pending_rollups.is_empty()); let state = store diff --git a/crates/statsai/src/cli/tests/sync/mod.rs b/crates/statsai/src/cli/tests/sync/mod.rs index 9ac0fcd..5905dbc 100644 --- a/crates/statsai/src/cli/tests/sync/mod.rs +++ b/crates/statsai/src/cli/tests/sync/mod.rs @@ -643,6 +643,9 @@ fn sync_local_verify_uses_sanitized_rollup_hashes() { store.rebuild_sync_rollups().expect("rebuild"); let target = "https://api.example.com/api/sync/batches".to_string(); + store + .record_sync_target_cache_health_support("http", &target, true) + .expect("receiver support"); let rollups: Vec<_> = store .all_sync_rollup_summaries() .expect("rollups") @@ -700,6 +703,9 @@ fn sync_local_verify_respects_project_sync_opt_in() { store.rebuild_sync_rollups().expect("rebuild"); let target = "https://api.example.com/api/sync/batches".to_string(); + store + .record_sync_target_cache_health_support("http", &target, true) + .expect("receiver support"); let rollups: Vec<_> = store .all_sync_rollup_summaries() .expect("rollups") diff --git a/crates/statsai/src/lib.rs b/crates/statsai/src/lib.rs index bd7852d..06157bb 100644 --- a/crates/statsai/src/lib.rs +++ b/crates/statsai/src/lib.rs @@ -372,6 +372,7 @@ mod tests { }, created_at: started_at, imported_at: started_at, + context: None, }; store.insert_event(&event).expect("legacy event"); assert_eq!(store.applied_pricing_ruleset_version().expect("meta"), None); diff --git a/crates/statsai/tests/cli_surface.rs b/crates/statsai/tests/cli_surface.rs index 2c0bea3..e858006 100644 --- a/crates/statsai/tests/cli_surface.rs +++ b/crates/statsai/tests/cli_surface.rs @@ -114,3 +114,10 @@ fn schema_quota_window_projection_matches_golden() { let golden = include_str!("cli_surface/schema-quota-window-projection.json"); assert_eq!(captured, golden); } + +#[test] +fn schema_cache_report_matches_golden() { + let captured = run_statsai(&["schema", "cache-report"]); + let golden = include_str!("cli_surface/schema-cache-report.json"); + assert_eq!(captured, golden); +} diff --git a/crates/statsai/tests/cli_surface/help.txt b/crates/statsai/tests/cli_surface/help.txt index 7d72338..9183926 100644 --- a/crates/statsai/tests/cli_surface/help.txt +++ b/crates/statsai/tests/cli_surface/help.txt @@ -59,6 +59,7 @@ Commands: monthly Show usage for the last 30 days all-time Show all stored usage range Show usage for an explicit date range + cache Show prompt-cache reuse, suspected cache losses, and context per call help Print this message or the help of the given subcommand(s) Options: @@ -115,6 +116,24 @@ Options: --verbose Show source paths and reasoning tokens --subscriptions Include subscription-value rows -h, --help Print help +===== statsai report cache --help ===== +Show prompt-cache reuse, suspected cache losses, and context per call + +Usage: statsai report cache [OPTIONS] + +Options: + --from Range start (YYYY-MM-DD or RFC3339). Date-only values are UTC days. Defaults to the last 7 days + --store Path to SQLite store + --device-id Device identifier for multi-device sync + --to Range end (YYYY-MM-DD or RFC3339). Defaults to now + --all Report all stored history + --provider Provider filter + --account Provider account id, or `unassigned` + --session Session id (session_…) or a prefix of one + --timeline Add local 10-minute buckets of calls + --details List every call with its verdict and evidence + --json Output as JSON + -h, --help Print help ===== statsai sessions --help ===== List local session rollups @@ -1119,6 +1138,7 @@ Usage: statsai schema [OPTIONS] Commands: sync-batch Print the sync_batch.v6 JSON Schema quota-window-projection Print the quota_window_sync_projection.v1 JSON Schema + cache-report Print the cache_report.v1 JSON Schema help Print this message or the help of the given subcommand(s) Options: @@ -1139,6 +1159,15 @@ Print the quota_window_sync_projection.v1 JSON Schema Usage: statsai schema quota-window-projection [OPTIONS] +Options: + --store Path to SQLite store + --device-id Device identifier for multi-device sync + -h, --help Print help +===== statsai schema cache-report --help ===== +Print the cache_report.v1 JSON Schema + +Usage: statsai schema cache-report [OPTIONS] + Options: --store Path to SQLite store --device-id Device identifier for multi-device sync diff --git a/crates/statsai/tests/cli_surface/schema-cache-report.json b/crates/statsai/tests/cli_surface/schema-cache-report.json new file mode 100644 index 0000000..0bf1f91 --- /dev/null +++ b/crates/statsai/tests/cli_surface/schema-cache-report.json @@ -0,0 +1,863 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CacheReport", + "description": "The prompt-cache report for a period, as `statsai report cache --json` and the local API print it.", + "type": "object", + "required": [ + "boundaries", + "days", + "detector_version", + "diagnostics", + "filters", + "gap_histogram", + "label", + "providers", + "schema_version", + "totals", + "until" + ], + "properties": { + "boundaries": { + "$ref": "#/definitions/CacheBoundaryCounts" + }, + "days": { + "description": "UTC days, matching the daily summaries that sync.", + "type": "array", + "items": { + "$ref": "#/definitions/CacheDayRow" + } + }, + "details": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/AnalyzedCacheCall" + } + }, + "detector_version": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "diagnostics": { + "description": "Detector counts across the providers it covers.", + "allOf": [ + { + "$ref": "#/definitions/CacheHealthV1" + } + ] + }, + "filters": { + "$ref": "#/definitions/CacheReportFilters" + }, + "gap_histogram": { + "type": "array", + "items": { + "$ref": "#/definitions/CacheGapRow" + } + }, + "label": { + "type": "string" + }, + "providers": { + "type": "array", + "items": { + "$ref": "#/definitions/CacheProviderRow" + } + }, + "schema_version": { + "type": "string" + }, + "since": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "timeline": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/CacheTimelineRow" + } + }, + "totals": { + "description": "Token totals across every provider in scope.", + "allOf": [ + { + "$ref": "#/definitions/CacheTokenTotals" + } + ] + }, + "until": { + "type": "string", + "format": "date-time" + } + }, + "definitions": { + "AnalyzedCacheCall": { + "description": "One model call and its verdict, or one unclassifiable block of usage.", + "type": "object", + "required": [ + "cache_read_tokens", + "call_index", + "completed_at", + "event_id", + "input_tokens", + "provider", + "requests", + "session_id", + "source_id", + "verdict" + ], + "properties": { + "agent_hash": { + "type": [ + "string", + "null" + ] + }, + "cache_creation_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "cache_read_tokens": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "call_index": { + "description": "Position inside an aggregated event; zero for single-call events.", + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "completed_at": { + "type": "string", + "format": "date-time" + }, + "event_id": { + "type": "string" + }, + "input_tokens": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "model": { + "type": [ + "string", + "null" + ] + }, + "provider": { + "type": "string" + }, + "provider_account_id": { + "type": [ + "string", + "null" + ] + }, + "requested_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "requests": { + "description": "Model calls this entry stands for: one, or an aggregated event's count.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "session_id": { + "type": "string" + }, + "source_id": { + "type": "string" + }, + "verdict": { + "$ref": "#/definitions/CacheCallVerdict" + }, + "write_ttl": { + "anyOf": [ + { + "$ref": "#/definitions/CacheWriteTtl" + }, + { + "type": "null" + } + ] + } + } + }, + "CacheBoundaryCounts": { + "description": "Boundary counts by reason. Local only: the synced object carries the total.", + "type": "object", + "required": [ + "compaction", + "evidence_gap", + "model_change" + ], + "properties": { + "compaction": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "evidence_gap": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "model_change": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + } + }, + "CacheBoundaryReason": { + "oneOf": [ + { + "description": "The model changed. Prompt caches are per model.", + "type": "string", + "enum": [ + "model_change" + ] + }, + { + "description": "The provider compacted the conversation, which rewrites the prompt.", + "type": "string", + "enum": [ + "compaction" + ] + }, + { + "description": "The previous usage in this stream could not be classified.", + "type": "string", + "enum": [ + "evidence_gap" + ] + } + ] + }, + "CacheCallClass": { + "description": "How the detector classified one model call.", + "oneOf": [ + { + "description": "The first call observed in its stream. Nothing earlier to compare with.", + "type": "string", + "enum": [ + "first_observed" + ] + }, + { + "description": "The comparison restarts here; see [`CacheBoundaryReason`].", + "type": "string", + "enum": [ + "boundary" + ] + }, + { + "description": "The previous call left too little reusable context to judge a loss.", + "type": "string", + "enum": [ + "below_threshold" + ] + }, + { + "description": "Compared with the previous call in its stream.", + "type": "string", + "enum": [ + "comparable" + ] + }, + { + "description": "The call neither read nor wrote the cache, so it did not use it, as when a custom endpoint serves the request. Not judged, and it leaves nothing reusable for the next call.", + "type": "string", + "enum": [ + "uncached" + ] + }, + { + "description": "Usage that cannot be placed in order or split into calls.", + "type": "string", + "enum": [ + "unclassifiable" + ] + } + ] + }, + "CacheCallVerdict": { + "type": "object", + "required": [ + "class" + ], + "properties": { + "baseline_tokens": { + "description": "Reusable context the previous call left, bounded by this call's input.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "boundary": { + "anyOf": [ + { + "$ref": "#/definitions/CacheBoundaryReason" + }, + { + "type": "null" + } + ] + }, + "class": { + "$ref": "#/definitions/CacheCallClass" + }, + "gap_seconds": { + "description": "Seconds since the previous call in the stream completed.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "gap_timing": { + "anyOf": [ + { + "$ref": "#/definitions/CacheGapTiming" + }, + { + "type": "null" + } + ] + }, + "loss": { + "anyOf": [ + { + "$ref": "#/definitions/CacheLoss" + }, + { + "type": "null" + } + ] + }, + "missed_cost_micro_usd": { + "description": "What the missed reuse cost at API prices: the call's cost minus its cost had the missed tokens been cache reads. Set by the caller that prices calls; `None` when the model's pricing is unknown.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "missed_tokens": { + "description": "For a suspected loss, the reusable context it did not read from the cache: the baseline minus its cached reads.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "previous_model": { + "description": "The model before a model change.", + "type": [ + "string", + "null" + ] + }, + "previous_write_ttl": { + "description": "Cache lifetime the previous call's writes asked for, when recorded.", + "anyOf": [ + { + "$ref": "#/definitions/CacheWriteTtl" + }, + { + "type": "null" + } + ] + }, + "unclassifiable": { + "anyOf": [ + { + "$ref": "#/definitions/CacheUnclassifiableReason" + }, + { + "type": "null" + } + ] + } + } + }, + "CacheDayRow": { + "type": "object", + "required": [ + "day", + "totals" + ], + "properties": { + "day": { + "type": "string" + }, + "diagnostics": { + "anyOf": [ + { + "$ref": "#/definitions/CacheHealthV1" + }, + { + "type": "null" + } + ] + }, + "totals": { + "$ref": "#/definitions/CacheTokenTotals" + } + } + }, + "CacheGapRow": { + "type": "object", + "required": [ + "comparable", + "label", + "losses", + "min_minutes", + "missed_tokens" + ], + "properties": { + "comparable": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "label": { + "type": "string" + }, + "losses": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "max_minutes": { + "description": "Exclusive upper bound; `None` for the open last bin.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "min_minutes": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "missed_tokens": { + "description": "Reusable context the bin's suspected losses did not read from the cache.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + } + }, + "CacheGapTiming": { + "description": "What a request gap was measured between.", + "oneOf": [ + { + "description": "From the previous response to the record that sent this request.", + "type": "string", + "enum": [ + "request_start" + ] + }, + { + "description": "Between the two recorded responses. Includes this call's own run time, so it overstates idle time.", + "type": "string", + "enum": [ + "recorded_events" + ] + } + ] + }, + "CacheHealthV1": { + "description": "Bounded prompt-cache diagnostics for one daily summary.\n\nOnly counts leave the device: no timestamps, ids, or per-call values. Token totals stay on the summary's `usage`, so this adds only what they cannot say. `requests` equals `unclassifiable + first_observed + boundaries + below_threshold + uncached + comparable`, and the gap bins add up to `comparable`, `losses`, and `missed_tokens`. Monthly rollups embed every day's object, so the fields most days leave at zero are left out then.", + "type": "object", + "required": [ + "below_threshold", + "boundaries", + "comparable", + "first_cold", + "first_observed", + "full_losses", + "gap_comparable", + "gap_losses", + "losses", + "requests", + "timing_estimated", + "unclassifiable", + "version", + "write_split_unavailable_input_tokens" + ], + "properties": { + "below_threshold": { + "description": "Calls whose predecessor left too little reusable context to judge.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "boundaries": { + "description": "Calls after a model change, compaction, or unclassifiable usage.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "comparable": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "first_cold": { + "description": "First calls that read nothing from the cache.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "first_observed": { + "description": "First calls of their stream.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "full_losses": { + "description": "Suspected losses that read nothing from the cache.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "gap_comparable": { + "description": "Comparable calls by gap since the previous call, one count per bin of [`super::CACHE_GAP_BIN_UPPER_MINUTES`]: `<1m` first, `60m+` last.", + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + }, + "gap_losses": { + "description": "Suspected losses in the same bins.", + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + }, + "gap_missed_tokens": { + "description": "Missed tokens in the same bins.", + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + }, + "losses": { + "description": "Suspected cache losses among comparable calls.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "missed_cost_micro_usd": { + "description": "What that missed reuse cost at API prices, in micro-USD.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "missed_tokens": { + "description": "Reusable context suspected losses did not read from the cache.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "requests": { + "description": "Model calls of providers the detector covers.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "timing_estimated": { + "description": "Comparable calls whose gap was measured between recorded responses rather than from a recorded request start.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "uncached": { + "description": "Calls that neither read nor wrote the cache, so did not use it.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "unclassifiable": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "unpriced_losses": { + "description": "Suspected losses on models without known pricing, left out of `missed_cost_micro_usd`.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "version": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "write_split_unavailable_input_tokens": { + "description": "Ordinary input from calls whose provider did not report cache writes, so it may include writes.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + } + }, + "CacheLoss": { + "oneOf": [ + { + "description": "No cached reads remained.", + "type": "string", + "enum": [ + "full" + ] + }, + { + "description": "Some cached context was still read.", + "type": "string", + "enum": [ + "partial" + ] + } + ] + }, + "CacheProviderRow": { + "type": "object", + "required": [ + "provider", + "totals" + ], + "properties": { + "diagnostics": { + "description": "`None` for providers the detector does not cover.", + "anyOf": [ + { + "$ref": "#/definitions/CacheHealthV1" + }, + { + "type": "null" + } + ] + }, + "provider": { + "type": "string" + }, + "totals": { + "$ref": "#/definitions/CacheTokenTotals" + } + } + }, + "CacheReportFilters": { + "type": "object", + "properties": { + "account": { + "type": [ + "string", + "null" + ] + }, + "provider": { + "type": [ + "string", + "null" + ] + }, + "session": { + "type": [ + "string", + "null" + ] + } + } + }, + "CacheTimelineRow": { + "description": "One local-time bucket of model calls, keyed by when each call completed.", + "type": "object", + "required": [ + "boundaries", + "cache_read_tokens", + "first_observed", + "logical_input_tokens", + "losses", + "missed_tokens", + "requests", + "start" + ], + "properties": { + "boundaries": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "cache_hit_ratio": { + "type": [ + "number", + "null" + ], + "format": "double" + }, + "cache_read_tokens": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "first_observed": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "logical_input_tokens": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "losses": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "missed_tokens": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "requests": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "start": { + "description": "Bucket start in local time, RFC 3339 with offset.", + "type": "string" + } + } + }, + "CacheTokenTotals": { + "type": "object", + "required": [ + "cache_creation_tokens", + "cache_read_tokens", + "input_tokens", + "logical_input_tokens", + "requests" + ], + "properties": { + "avg_logical_input_per_request": { + "type": [ + "number", + "null" + ], + "format": "double" + }, + "cache_creation_tokens": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "cache_hit_ratio": { + "description": "Cache reads over logical input.", + "type": [ + "number", + "null" + ], + "format": "double" + }, + "cache_read_tokens": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "input_tokens": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "logical_input_tokens": { + "description": "Ordinary input, cache writes, and cache reads: the context processed.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "requests": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + } + }, + "CacheUnclassifiableReason": { + "oneOf": [ + { + "description": "The record had no timestamp, so its place in the stream is unknown.", + "type": "string", + "enum": [ + "timestamp_missing" + ] + }, + { + "description": "Several calls were recorded as one total with no per-call breakdown.", + "type": "string", + "enum": [ + "aggregated" + ] + }, + { + "description": "The provider reported no input or cache counts.", + "type": "string", + "enum": [ + "no_input_telemetry" + ] + }, + { + "description": "Another call in the stream has the same timestamp and no recorded order.", + "type": "string", + "enum": [ + "ambiguous_order" + ] + } + ] + }, + "CacheWriteTtl": { + "description": "The cache lifetime a call's writes asked for, as Claude Code records it.", + "type": "string", + "enum": [ + "five_minutes", + "one_hour", + "mixed" + ] + } + } +} diff --git a/crates/statsai/tests/cli_surface/schema-sync-batch.json b/crates/statsai/tests/cli_surface/schema-sync-batch.json index 2610831..f809ac4 100644 --- a/crates/statsai/tests/cli_surface/schema-sync-batch.json +++ b/crates/statsai/tests/cli_surface/schema-sync-batch.json @@ -553,6 +553,148 @@ "custom" ] }, + "CacheHealthV1": { + "description": "Bounded prompt-cache diagnostics for one daily summary.\n\nOnly counts leave the device: no timestamps, ids, or per-call values. Token totals stay on the summary's `usage`, so this adds only what they cannot say. `requests` equals `unclassifiable + first_observed + boundaries + below_threshold + uncached + comparable`, and the gap bins add up to `comparable`, `losses`, and `missed_tokens`. Monthly rollups embed every day's object, so the fields most days leave at zero are left out then.", + "type": "object", + "required": [ + "below_threshold", + "boundaries", + "comparable", + "first_cold", + "first_observed", + "full_losses", + "gap_comparable", + "gap_losses", + "losses", + "requests", + "timing_estimated", + "unclassifiable", + "version", + "write_split_unavailable_input_tokens" + ], + "properties": { + "below_threshold": { + "description": "Calls whose predecessor left too little reusable context to judge.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "boundaries": { + "description": "Calls after a model change, compaction, or unclassifiable usage.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "comparable": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "first_cold": { + "description": "First calls that read nothing from the cache.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "first_observed": { + "description": "First calls of their stream.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "full_losses": { + "description": "Suspected losses that read nothing from the cache.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "gap_comparable": { + "description": "Comparable calls by gap since the previous call, one count per bin of [`super::CACHE_GAP_BIN_UPPER_MINUTES`]: `<1m` first, `60m+` last.", + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + }, + "gap_losses": { + "description": "Suspected losses in the same bins.", + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + }, + "gap_missed_tokens": { + "description": "Missed tokens in the same bins.", + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + }, + "losses": { + "description": "Suspected cache losses among comparable calls.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "missed_cost_micro_usd": { + "description": "What that missed reuse cost at API prices, in micro-USD.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "missed_tokens": { + "description": "Reusable context suspected losses did not read from the cache.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "requests": { + "description": "Model calls of providers the detector covers.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "timing_estimated": { + "description": "Comparable calls whose gap was measured between recorded responses rather than from a recorded request start.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "uncached": { + "description": "Calls that neither read nor wrote the cache, so did not use it.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "unclassifiable": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "unpriced_losses": { + "description": "Suspected losses on models without known pricing, left out of `missed_cost_micro_usd`.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "version": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "write_split_unavailable_input_tokens": { + "description": "Ordinary input from calls whose provider did not report cache writes, so it may include writes.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + } + }, "CodeChangeMetric": { "type": "object", "required": [ @@ -2151,6 +2293,17 @@ "format": "uint64", "minimum": 0.0 }, + "cache_health": { + "description": "Prompt-cache diagnostics, for providers whose events record call order.", + "anyOf": [ + { + "$ref": "#/definitions/CacheHealthV1" + }, + { + "type": "null" + } + ] + }, "cache_hit_ratio": { "anyOf": [ { diff --git a/crates/statsai/tests/report_cli.rs b/crates/statsai/tests/report_cli.rs index b139515..4ca7bc6 100644 --- a/crates/statsai/tests/report_cli.rs +++ b/crates/statsai/tests/report_cli.rs @@ -89,6 +89,7 @@ fn test_event( }, created_at: started_at, imported_at: started_at, + context: None, } }