diff --git a/README.md b/README.md index 8982bb7..1670ac0 100644 --- a/README.md +++ b/README.md @@ -156,13 +156,24 @@ dashboard. | Parse evidence and local source paths | Project labels and repository anchors | | Source text, diffs, file paths, and commit messages | Privacy-safe numeric code-change metrics | | — | Opt-in private task snapshots and verifications, including bounded task titles, summary previews, and todo excerpts | +| Session prompts, raw session ids, and message text | Opt-in session rollups: totals, duration, message counts, and a bounded title | Raw usage events and complete archived conversation records stay local and are never included in hosted sync. StatsAI does not upload full prompts, full responses, or raw provider logs. When hosted task sync is explicitly enabled with `statsai sync --include-tasks`, the current `sync_batch.v5` payload may include bounded conversation-derived task titles, summary previews, and todo -excerpts. You can inspect the exact sync contract in +excerpts. Hosted session sync is separate and off by default. +`statsai sync --include-sessions` adds `sessions` rows to `sync_batch.v6`: +hashed session ids, token and request totals, active time and wall-clock span, +message counts, and a bounded title. Active time is the union of the agent's +turns, each from the prompt until its last message; idle time between turns is +not counted. Titles come from the provider's own session name (Codex thread +names, Claude Code and OpenCode session titles) or a local task title. Cursor's +usage export names no session, so its local rows are not sessions, and a session +with neither a project nor a title is not kept. +`--exclude-sessions` stops sending them and leaves sessions already hosted in +place. You can inspect the exact sync contract in [`docs/sync-contract.md`](docs/sync-contract.md) and verify the resolved sync target with: diff --git a/crates/statsai-adapters/src/cache.rs b/crates/statsai-adapters/src/cache.rs index 370c00d..e3b9edd 100644 --- a/crates/statsai-adapters/src/cache.rs +++ b/crates/statsai-adapters/src/cache.rs @@ -11,11 +11,19 @@ pub(crate) const SCAN_CACHE_SIGNATURE_VERSION: &str = "scan-cache.v1"; // activity-invocations.v34: Codex usage skips repeated token_count totals and // counts token_usage_record lines, including compaction, instead of the // token_count each one pairs with. -pub(crate) const CODEX_SCAN_CACHE_PARSER_REVISION: &str = "activity-invocations.v34"; -pub(crate) const CLAUDE_SCAN_CACHE_PARSER_REVISION: &str = "activity-invocations.v28"; -// activity-invocations.v20: session rows report their message count as requests and -// are priced per message, so long-context tiers stop being decided session-wide. -pub(crate) const OPENCODE_SCAN_CACHE_PARSER_REVISION: &str = "activity-invocations.v20"; +// session-metadata.v35: Codex events carry thread names, sub-agents their +// parent's name, and turns without a reported duration end at their last work. +// session-metadata.v36: sub-agents report the parent they borrow a name from, +// so the name is joined when the session is built. +pub(crate) const CODEX_SCAN_CACHE_PARSER_REVISION: &str = "session-metadata.v36"; +// session-metadata.v30: Claude user-message counts include only typed prompts, +// not tool results, meta lines, or compaction summaries. (v29: events carry the +// session's custom or AI title and the prompt that started their turn.) +pub(crate) const CLAUDE_SCAN_CACHE_PARSER_REVISION: &str = "session-metadata.v30"; +// session-metadata.v21: events carry each session's user and assistant message +// counts. (v20: session rows report their message count as requests and are +// priced per message, so long-context tiers stop being decided session-wide.) +pub(crate) const OPENCODE_SCAN_CACHE_PARSER_REVISION: &str = "session-metadata.v21"; // Revisit Grok sessions whose Fast request was left unpriced because modelsUsed // also retained a selected standard model that made no inference. pub(crate) const GROK_BUILD_SCAN_CACHE_PARSER_REVISION: &str = "grok-fast-pricing.v23"; diff --git a/crates/statsai-adapters/src/claude/mod.rs b/crates/statsai-adapters/src/claude/mod.rs index 308420a..b7c20e8 100644 --- a/crates/statsai-adapters/src/claude/mod.rs +++ b/crates/statsai-adapters/src/claude/mod.rs @@ -205,6 +205,7 @@ pub(crate) fn scan_claude_source( let event_files = claude_jsonl_candidates(&projects, &cache_namespaces)?; let mut scanned_event_cache_keys = HashSet::new(); let mut seen = EventDedupIndex::new(); + let mut session_titles = ClaudeSessionTitles::default(); { let mut ctx = FileParseContext { adapter, @@ -220,9 +221,17 @@ pub(crate) fn scan_claude_source( } ctx.scan.diagnostics.files_scanned += 1; scanned_event_cache_keys.insert(candidate.cache_key.clone()); - parse_claude_file(&mut ctx, &projects, &session_projects, &candidate.path)?; + parse_claude_file( + &mut ctx, + &projects, + &session_projects, + &mut session_titles, + &candidate.path, + )?; } } + let session_names = session_titles.apply(&mut scan.events); + scan.session_names.extend(session_names); if let Some(candidate) = claude_stats_cache_candidate(&root, &cache_namespaces) { if options.should_scan(&candidate.cache_key) { diff --git a/crates/statsai-adapters/src/claude/parse.rs b/crates/statsai-adapters/src/claude/parse.rs index 67fddb5..d27275a 100644 --- a/crates/statsai-adapters/src/claude/parse.rs +++ b/crates/statsai-adapters/src/claude/parse.rs @@ -5,20 +5,21 @@ use crate::model::{ }; use crate::ProviderAdapter; use crate::{ - fallback_session_id, file_modified_timestamp, infer_missing_output, metadata_only_privacy, - number_at_any, project_context_from_path_fallback, push_deduped, read_bounded_jsonl_line, - resolve_project_context, resolve_project_context_cached, stats_cache_date_end, - timestamp_from_nested_value, timestamp_from_scalar, usage_event, usd_to_micro_usd, - value_as_u64, AdapterScan, BoundedLineRead, DuplicateSelection, EventDeduplication, - FileParseContext, ProjectContextCache, ProviderEventParts, ScanOptions, MAX_JSONL_RECORD_BYTES, + fallback_session_id, file_modified_timestamp, infer_missing_output, message_count_runtime, + metadata_only_privacy, number_at_any, project_context_from_path_fallback, push_deduped, + read_bounded_jsonl_line, resolve_project_context, resolve_project_context_cached, + stats_cache_date_end, timestamp_from_nested_value, timestamp_from_scalar, usage_event, + usd_to_micro_usd, value_as_u64, AdapterScan, BoundedLineRead, DuplicateSelection, + EventDeduplication, FileParseContext, ProjectContextCache, ProviderEventParts, ScanOptions, + MAX_JSONL_RECORD_BYTES, }; use anyhow::{Context, Result}; use chrono::Utc; use serde_json::Value; use statsai_core::{ - canonical_display, expand_home_path, hash_text, summary_id, Confidence, EventSource, - IdentitySource, ParseEvidence, ProjectInfo, SourceKind, SourceLocation, SummaryMetadata, - UsageCounts, UsageSummary, USAGE_SUMMARY_SCHEMA_VERSION, + canonical_display, expand_home_path, hash_text, provider_session_name, summary_id, Confidence, + EventSource, IdentitySource, ParseEvidence, ProjectInfo, SourceKind, SourceLocation, + SummaryMetadata, UsageCounts, UsageSummary, USAGE_SUMMARY_SCHEMA_VERSION, }; use statsai_pricing::{estimate_cost_at, pricing_changes_between, unknown_cost}; use std::collections::HashMap; @@ -30,6 +31,7 @@ pub(crate) fn parse_claude_file( ctx: &mut FileParseContext<'_, ClaudeCodeAdapter>, projects: &Path, session_projects: &HashMap, + session_titles: &mut ClaudeSessionTitles, path: &Path, ) -> Result<()> { let file = File::open(path).with_context(|| format!("read {}", path.display()))?; @@ -43,6 +45,11 @@ pub(crate) fn parse_claude_file( let mut line_bytes = Vec::new(); let mut index = 0usize; + let mut pending_user_messages = HashMap::::new(); + // Later snapshots of one provider record replace the event. User lines + // counted on the first snapshot have to ride along with that replacement. + let mut last_usage_users = HashMap::::new(); + let mut turn_started_at = HashMap::>::new(); loop { let line_status = read_bounded_jsonl_line(&mut reader, &mut line_bytes, MAX_JSONL_RECORD_BYTES)?; @@ -76,18 +83,52 @@ pub(crate) fn parse_claude_file( if reasoning.raw.is_some() { current_reasoning = reasoning; } + let session_raw = claude_session_raw(&value, path); + if let Some((rank, title)) = claude_session_title_from_value(&value) { + session_titles.remember(session_raw, rank, title); + continue; + } + if claude_record_is_prompt(&value) { + if let Some(prompted_at) = timestamp_from_nested_value(&value) { + turn_started_at.insert(session_raw.clone(), prompted_at); + } + } let Some(usage_value) = value .pointer("/message/usage") .or_else(|| value.get("usage")) else { + if claude_record_is_prompt(&value) { + *pending_user_messages.entry(session_raw).or_default() += 1; + } continue; }; ctx.scan.diagnostics.candidate_usage_rows += 1; let usage = claude_usage_counts_from_value(usage_value); if usage.computed_total() == 0 { + if claude_record_is_prompt(&value) { + *pending_user_messages.entry(session_raw).or_default() += 1; + } ctx.scan.diagnostics.skipped_zero_events += 1; continue; } + let record_id = claude_provider_record_id(&value); + let fresh_users = pending_user_messages.remove(&session_raw).unwrap_or(0); + let user_messages = match record_id.as_deref() { + Some(record_id) => { + let carried = last_usage_users + .get(&session_raw) + .filter(|(previous, _)| previous == record_id) + .map(|(_, users)| *users) + .unwrap_or(0); + let users = carried.saturating_add(fresh_users); + last_usage_users.insert(session_raw.clone(), (record_id.to_string(), users)); + users + } + None => { + last_usage_users.remove(&session_raw); + fresh_users + } + }; let (timestamp, timestamp_inferred) = timestamp_from_nested_value(&value) .map(|timestamp| (timestamp, false)) .unwrap_or((fallback_timestamp, true)); @@ -106,12 +147,6 @@ pub(crate) fn parse_claude_file( let project = claude_project_context_from_value(&value, indexed_project_metadata, &mut project_cache) .or_else(|| fallback_project.clone()); - let session_raw = value - .get("sessionId") - .or_else(|| value.get("session_id")) - .and_then(Value::as_str) - .map(ToOwned::to_owned) - .unwrap_or_else(|| fallback_session_id(path)); // Claude Code rewrites one streamed response as several records that // share a provider-record identity while cumulative usage grows, so the // last snapshot for that identity is the authoritative one. Records @@ -128,7 +163,8 @@ pub(crate) fn parse_claude_file( ) }, ); - let event = usage_event( + let prompted_at = turn_started_at.get(&session_raw).copied(); + let mut event = usage_event( ctx.adapter, ctx.source, ctx.options, @@ -139,7 +175,7 @@ pub(crate) fn parse_claude_file( duration_seconds: None, model, usage, - runtime: None, + runtime: message_count_runtime(user_messages, 1, 0), session_raw, project, event_kind: "claude_message_usage", @@ -152,6 +188,7 @@ pub(crate) fn parse_claude_file( dedupe_salt: None, }, ); + event.session.turn_started_at = prompted_at.filter(|prompted_at| *prompted_at <= timestamp); push_deduped(ctx.scan, ctx.seen, event, duplicate_selection); } @@ -167,6 +204,108 @@ pub(crate) fn parse_claude_file( Ok(()) } +/// Session names found across one source's transcripts. +/// +/// A resumed or forked session copies earlier messages into a new transcript, +/// and those events dedupe onto the positions of the file that first held +/// them. Titles are therefore applied once the whole source is read, by session +/// hash, rather than to the events of the file that named the session. +#[derive(Default)] +pub(crate) struct ClaudeSessionTitles(HashMap); + +impl ClaudeSessionTitles { + fn remember(&mut self, session_raw: String, rank: u8, title: String) { + let replaces = self + .0 + .get(&session_raw) + .is_none_or(|(current, _)| rank >= *current); + if replaces { + self.0.insert(session_raw, (rank, title)); + } + } + + /// Stamps the titles on this scan's events and returns them as session + /// names, which also reach sessions whose events this scan did not read. + pub(crate) fn apply( + self, + events: &mut [statsai_core::UsageEvent], + ) -> Vec { + let titles_by_hash = self + .0 + .into_iter() + .map(|(session_raw, (_, title))| (hash_text(&session_raw), title)) + .collect::>(); + for event in events.iter_mut() { + if let Some(title) = event + .session + .local_session_id_hash + .as_deref() + .and_then(|hash| titles_by_hash.get(hash)) + { + event.session.title = Some(title.clone()); + } + } + titles_by_hash + .into_iter() + .map(|(local_session_id_hash, title)| statsai_core::SessionName { + local_session_id_hash, + title, + parent_local_session_id_hash: None, + }) + .collect() + } +} + +/// Claude Code names a session with `custom-title` records, which a rename +/// also writes, and occasionally `ai-title`. The later record of the stronger +/// kind wins; the rank says which kind that is. +fn claude_session_title_from_value(value: &Value) -> Option<(u8, String)> { + let (rank, key) = match value.get("type").and_then(Value::as_str)? { + "custom-title" => (1, "customTitle"), + "ai-title" => (0, "aiTitle"), + _ => return None, + }; + let title = provider_session_name(value.get(key).and_then(Value::as_str), 90)?; + Some((rank, title)) +} + +fn claude_session_raw(value: &Value, path: &Path) -> String { + value + .get("sessionId") + .or_else(|| value.get("session_id")) + .and_then(Value::as_str) + .map(ToOwned::to_owned) + .unwrap_or_else(|| fallback_session_id(path)) +} + +/// A prompt starts a turn: typed text, an image, or a slash command. Tool +/// results arrive as `user` records too, but they continue the turn, as do +/// meta records and compaction summaries. +fn claude_record_is_prompt(value: &Value) -> bool { + if !claude_record_is_user(value) + || value.get("toolUseResult").is_some() + || value.get("isMeta").and_then(Value::as_bool) == Some(true) + || value.get("isCompactSummary").and_then(Value::as_bool) == Some(true) + { + return false; + } + match value.pointer("/message/content") { + Some(Value::String(text)) => !text.trim().is_empty(), + Some(Value::Array(blocks)) => blocks.iter().any(|block| { + matches!( + block.get("type").and_then(Value::as_str), + Some("text" | "image") + ) + }), + _ => false, + } +} + +fn claude_record_is_user(value: &Value) -> bool { + value.get("type").and_then(Value::as_str) == Some("user") + || value.pointer("/message/role").and_then(Value::as_str) == Some("user") +} + pub(crate) fn claude_provider_record_id(value: &Value) -> Option { let message_id = value .pointer("/message/id") diff --git a/crates/statsai-adapters/src/claude/tests/parse.rs b/crates/statsai-adapters/src/claude/tests/parse.rs index c440efe..9cf184e 100644 --- a/crates/statsai-adapters/src/claude/tests/parse.rs +++ b/crates/statsai-adapters/src/claude/tests/parse.rs @@ -498,6 +498,100 @@ fn claude_streaming_snapshots_keep_the_final_usage_for_one_request() { ); } +#[test] +fn claude_user_counts_stay_on_the_kept_streaming_snapshot() { + let dir = tempfile::tempdir().expect("tempdir"); + let projects = dir.path().join("projects"); + std::fs::create_dir_all(&projects).expect("projects"); + let mut file = File::create(projects.join("session.jsonl")).expect("session"); + writeln!( + file, + r#"{{"type":"user","timestamp":"2026-08-05T14:51:09.000Z","sessionId":"session-1","message":{{"role":"user","content":"hello"}}}}"# + ) + .expect("user"); + for (timestamp, uuid, output) in [ + ("2026-08-05T14:51:09.702Z", "record-1", 10), + ("2026-08-05T14:51:11.102Z", "record-2", 40), + ] { + writeln!( + file, + r#"{{"timestamp":"{timestamp}","sessionId":"session-1","uuid":"{uuid}","requestId":"request-1","message":{{"id":"message-1","role":"assistant","model":"claude-opus-5","usage":{{"input_tokens":2,"output_tokens":{output}}}}}}}"# + ) + .expect("snapshot"); + } + writeln!( + file, + r#"{{"type":"user","timestamp":"2026-08-05T14:51:12.000Z","sessionId":"session-1","message":{{"role":"user","content":"again"}}}}"# + ) + .expect("second user"); + writeln!( + file, + r#"{{"timestamp":"2026-08-05T14:51:14.209Z","sessionId":"session-1","uuid":"record-3","requestId":"request-2","message":{{"id":"message-2","role":"assistant","model":"claude-opus-5","usage":{{"input_tokens":3,"output_tokens":5}}}}}}"# + ) + .expect("second request"); + + let source = SourceLocation::local_adapter( + CLAUDE_CODE_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let scan = scan_claude_source(&ClaudeCodeAdapter, &source, &options()).expect("scan"); + + assert_eq!(scan.events.len(), 2); + let first = scan.events[0].runtime.as_ref().expect("first runtime"); + assert_eq!(first.user_messages, Some(1)); + assert_eq!(first.assistant_messages, Some(1)); + assert_eq!(scan.events[0].usage.output_tokens, Some(40)); + let second = scan.events[1].runtime.as_ref().expect("second runtime"); + assert_eq!(second.user_messages, Some(1)); + assert_eq!(second.assistant_messages, Some(1)); +} + +#[test] +fn claude_user_counts_skip_tool_results_meta_lines_and_compaction_summaries() { + let dir = tempfile::tempdir().expect("tempdir"); + let projects = dir.path().join("projects"); + std::fs::create_dir_all(&projects).expect("projects"); + let mut file = File::create(projects.join("session.jsonl")).expect("session"); + for line in [ + r#"{"type":"user","timestamp":"2026-08-05T14:51:00.000Z","sessionId":"session-1","isMeta":true,"message":{"role":"user","content":"Caveat"}}"#, + r#"{"type":"user","timestamp":"2026-08-05T14:51:01.000Z","sessionId":"session-1","isCompactSummary":true,"message":{"role":"user","content":"This session is being continued"}}"#, + r#"{"type":"user","timestamp":"2026-08-05T14:51:02.000Z","sessionId":"session-1","message":{"role":"user","content":"fix the parser"}}"#, + r#"{"timestamp":"2026-08-05T14:51:03.000Z","sessionId":"session-1","uuid":"record-1","requestId":"request-1","message":{"id":"message-1","role":"assistant","model":"claude-opus-5","usage":{"input_tokens":2,"output_tokens":10}}}"#, + r#"{"type":"user","timestamp":"2026-08-05T14:51:04.000Z","sessionId":"session-1","toolUseResult":{"stdout":"ok"},"message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"tool-1","content":"ok"}]}}"#, + r#"{"type":"user","timestamp":"2026-08-05T14:51:05.000Z","sessionId":"session-1","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"tool-2","content":"ok"}]}}"#, + r#"{"timestamp":"2026-08-05T14:51:06.000Z","sessionId":"session-1","uuid":"record-2","requestId":"request-2","message":{"id":"message-2","role":"assistant","model":"claude-opus-5","usage":{"input_tokens":3,"output_tokens":5}}}"#, + ] { + writeln!(file, "{line}").expect("line"); + } + + let source = SourceLocation::local_adapter( + CLAUDE_CODE_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let scan = scan_claude_source(&ClaudeCodeAdapter, &source, &options()).expect("scan"); + + // One typed prompt, followed by two tool results that are not prompts. + let user_messages = scan + .events + .iter() + .map(|event| { + event + .runtime + .as_ref() + .and_then(|runtime| runtime.user_messages) + .unwrap_or(0) + }) + .sum::(); + assert_eq!(scan.events.len(), 2); + assert_eq!(user_messages, 1); +} + #[test] fn claude_streaming_snapshots_with_equal_timestamps_resolve_by_source_line() { let dir = tempfile::tempdir().expect("tempdir"); @@ -1003,7 +1097,9 @@ fn claude_collects_effort_and_effective_speed_but_ignores_service_tier() { .expect("serialize model") .to_string() .contains("service_tier")); - assert!(scan.events[0].runtime.is_none()); + let runtime = scan.events[0].runtime.as_ref().expect("message counts"); + assert_eq!(runtime.assistant_messages, Some(1)); + assert!(runtime.latency_ms.is_none()); } #[test] @@ -1118,3 +1214,162 @@ fn claude_scan_separates_oversized_rows_from_malformed_ones() { ); assert_eq!(scan.events.len(), 1, "the usable row is still collected"); } + +#[test] +fn claude_session_titles_name_every_event_in_the_session() { + let dir = tempfile::tempdir().expect("tempdir"); + let projects = dir.path().join("projects"); + std::fs::create_dir_all(&projects).expect("projects"); + let mut file = File::create(projects.join("session.jsonl")).expect("session file"); + for line in [ + r#"{"type":"ai-title","aiTitle":"Generated name","sessionId":"titled"}"#, + r#"{"timestamp":"2026-05-01T00:00:00Z","sessionId":"titled","message":{"id":"m1","usage":{"input_tokens":1,"output_tokens":2}}}"#, + r#"{"timestamp":"2026-05-01T00:01:00Z","sessionId":"untitled","message":{"id":"m2","usage":{"input_tokens":1,"output_tokens":2}}}"#, + // A custom title outranks the generated one even though it comes later. + r#"{"type":"custom-title","customTitle":"Repair the session indexer","sessionId":"titled"}"#, + r#"{"type":"ai-title","aiTitle":"Later generated name","sessionId":"titled"}"#, + ] { + writeln!(file, "{line}").expect("write line"); + } + + let source = SourceLocation::local_adapter( + CLAUDE_CODE_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let scan = scan_claude_source(&ClaudeCodeAdapter, &source, &options()).expect("scan"); + + assert_eq!(scan.events.len(), 2); + let title_for = |session_raw: &str| { + let hash = statsai_core::hash_text(session_raw); + scan.events + .iter() + .find(|event| event.session.local_session_id_hash.as_deref() == Some(hash.as_str())) + .expect("event") + .session + .title + .clone() + }; + assert_eq!( + title_for("titled").as_deref(), + Some("Repair the session indexer") + ); + assert_eq!(title_for("untitled"), None); +} + +#[test] +fn claude_session_titles_reach_events_read_from_other_transcripts() { + let dir = tempfile::tempdir().expect("tempdir"); + let projects = dir.path().join("projects"); + std::fs::create_dir_all(&projects).expect("projects"); + // A resumed session's copied messages dedupe onto the transcript that + // first held them, while the title is written to the resumed transcript. + let mut original = File::create(projects.join("original.jsonl")).expect("original"); + writeln!( + original, + r#"{{"timestamp":"2026-05-01T00:00:00Z","sessionId":"resumed","message":{{"id":"m1","usage":{{"input_tokens":1,"output_tokens":2}}}}}}"# + ) + .expect("write original"); + let mut resumed = File::create(projects.join("resumed.jsonl")).expect("resumed"); + writeln!( + resumed, + r#"{{"type":"custom-title","customTitle":"Design tool/plugin","sessionId":"resumed"}}"# + ) + .expect("write resumed"); + + let source = SourceLocation::local_adapter( + CLAUDE_CODE_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let scan = scan_claude_source(&ClaudeCodeAdapter, &source, &options()).expect("scan"); + + assert_eq!(scan.events.len(), 1); + assert_eq!( + scan.events[0].session.title.as_deref(), + Some("Design tool/plugin") + ); +} + +#[test] +fn claude_events_carry_the_prompt_that_started_their_turn() { + let dir = tempfile::tempdir().expect("tempdir"); + let projects = dir.path().join("projects"); + std::fs::create_dir_all(&projects).expect("projects"); + let mut file = File::create(projects.join("session.jsonl")).expect("session file"); + for line in [ + r#"{"type":"user","timestamp":"2026-05-01T10:00:00Z","sessionId":"turns","message":{"role":"user","content":"Fix the parser"}}"#, + r#"{"type":"assistant","timestamp":"2026-05-01T10:01:00Z","sessionId":"turns","message":{"id":"m1","usage":{"input_tokens":1,"output_tokens":2}}}"#, + r#"{"type":"user","timestamp":"2026-05-01T10:02:00Z","sessionId":"turns","toolUseResult":{},"message":{"role":"user","content":[{"type":"tool_result","content":"ok"}]}}"#, + r#"{"type":"assistant","timestamp":"2026-05-01T10:03:00Z","sessionId":"turns","message":{"id":"m2","usage":{"input_tokens":1,"output_tokens":2}}}"#, + r#"{"type":"user","timestamp":"2026-05-01T15:00:00Z","sessionId":"turns","message":{"role":"user","content":[{"type":"text","text":"Now the tests"}]}}"#, + r#"{"type":"assistant","timestamp":"2026-05-01T15:04:00Z","sessionId":"turns","message":{"id":"m3","usage":{"input_tokens":1,"output_tokens":2}}}"#, + ] { + writeln!(file, "{line}").expect("write line"); + } + + let source = SourceLocation::local_adapter( + CLAUDE_CODE_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let scan = scan_claude_source(&ClaudeCodeAdapter, &source, &options()).expect("scan"); + let mut turns = scan + .events + .iter() + .map(|event| { + ( + event.session.started_at.to_rfc3339(), + event.session.turn_started_at.map(|at| at.to_rfc3339()), + ) + }) + .collect::>(); + turns.sort(); + let prompt = |hour: &str| Some(format!("2026-05-01T{hour}:00:00+00:00")); + assert_eq!( + turns, + vec![ + ("2026-05-01T10:01:00+00:00".to_string(), prompt("10")), + ("2026-05-01T10:03:00+00:00".to_string(), prompt("10")), + ("2026-05-01T15:04:00+00:00".to_string(), prompt("15")), + ] + ); +} + +#[test] +fn claude_title_only_transcripts_report_session_names() { + let dir = tempfile::tempdir().expect("tempdir"); + let projects = dir.path().join("projects"); + std::fs::create_dir_all(&projects).expect("projects"); + // A resumed transcript holding only the rename; the usage lives elsewhere. + let mut file = File::create(projects.join("resumed.jsonl")).expect("resumed"); + writeln!( + file, + r#"{{"type":"custom-title","customTitle":"Renamed session","sessionId":"renamed"}}"# + ) + .expect("write title"); + + let source = SourceLocation::local_adapter( + CLAUDE_CODE_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let scan = scan_claude_source(&ClaudeCodeAdapter, &source, &options()).expect("scan"); + assert!(scan.events.is_empty()); + assert_eq!( + scan.session_names, + vec![statsai_core::SessionName { + local_session_id_hash: statsai_core::hash_text("renamed"), + title: "Renamed session".to_string(), + parent_local_session_id_hash: None, + }] + ); +} diff --git a/crates/statsai-adapters/src/codex/mod.rs b/crates/statsai-adapters/src/codex/mod.rs index 32214ea..3b5befa 100644 --- a/crates/statsai-adapters/src/codex/mod.rs +++ b/crates/statsai-adapters/src/codex/mod.rs @@ -63,6 +63,18 @@ impl ProviderAdapter for CodexAdapter { codex_scan_candidates(source, self.version()) } + fn archive_scan_candidates(&self, source: &SourceLocation) -> Result> { + let Some(path_label) = source + .path_label + .as_deref() + .filter(|label| !label.is_empty()) + else { + return Ok(Vec::new()); + }; + let cache_namespaces = scan_cache_namespaces(source, self.version()); + codex_jsonl_candidates(source, Path::new(path_label), &cache_namespaces) + } + fn probe_verified_source_state( &self, source: &SourceLocation, @@ -123,11 +135,22 @@ pub(crate) fn scan_codex_source( let source_path = PathBuf::from(path_label); let root = codex_source_root(&source_path); let cache_namespaces = scan_cache_namespaces(source, adapter.version()); - let thread_titles = if options.should_collect_tasks() { - load_codex_thread_titles(&root) - } else { - HashMap::new() - }; + // Thread names title sessions as well as tasks, so they load either way. + // The index is re-read on every scan and its names go out as session + // names: a rename there changes no rollout, so the scan cache would keep + // every rollout's old title. + let thread_names = load_codex_thread_names(&root); + let thread_titles = codex_thread_titles_from_names(&thread_names); + scan.session_names + .extend( + thread_names + .into_iter() + .map(|(session_id, title)| statsai_core::SessionName { + local_session_id_hash: hash_text(&session_id), + title, + parent_local_session_id_hash: None, + }), + ); let mut indexed_candidates = Vec::new(); for (index, candidate) in codex_jsonl_candidates(source, &source_path, &cache_namespaces)? .into_iter() @@ -270,7 +293,15 @@ pub(crate) fn codex_scan_candidates( }; let source_path = PathBuf::from(path_label); let cache_namespaces = scan_cache_namespaces(source, adapter_version); - codex_jsonl_candidates(source, &source_path, &cache_namespaces) + let mut candidates = codex_jsonl_candidates(source, &source_path, &cache_namespaces)?; + // Renaming a thread rewrites only the index, so it is tracked as a file of + // its own: a change to it rescans the source for names without selecting + // any rollout. + let index_path = codex_source_root(&source_path).join(CODEX_SESSION_INDEX_FILE); + if index_path.is_file() { + candidates.push(scan_candidate(index_path, None, &cache_namespaces)); + } + Ok(candidates) } pub(crate) fn codex_jsonl_candidates( diff --git a/crates/statsai-adapters/src/codex/parse/mod.rs b/crates/statsai-adapters/src/codex/parse/mod.rs index 3b2edc2..054d0fa 100644 --- a/crates/statsai-adapters/src/codex/parse/mod.rs +++ b/crates/statsai-adapters/src/codex/parse/mod.rs @@ -39,6 +39,8 @@ pub(crate) fn parse_codex_file( let mut current_model_is_fallback = false; let mut current_project: Option = None; let mut current_title: Option = None; + // Events only: a sub-agent borrows its parent thread's name, but task + // titles keep their own prompt-based rules. let mut current_thread_id: Option = None; // Seeded from the file path, replaced by the session's own id as soon as // the `session_meta` line declares one. The embedded id is the same UUID @@ -341,22 +343,17 @@ pub(crate) fn parse_codex_file( session_raw = declared_session_id; } if collect_tasks { - current_thread_id = declared_session_id; - let session_id = current_thread_id - .clone() - .or_else(|| Some(session_raw.clone())); - current_title = value - .pointer("/payload/thread_name") - .and_then(Value::as_str) - .map(ToOwned::to_owned) - .or_else(|| { - session_id - .as_ref() - .and_then(|session_id| thread_titles.get(session_id)) - .cloned() - }) - .or_else(|| thread_titles.get(&session_raw).cloned()); + current_thread_id = declared_session_id.clone(); } + let session_id = declared_session_id.unwrap_or_else(|| session_raw.clone()); + current_title = value + .pointer("/payload/thread_name") + .and_then(Value::as_str) + .map(ToOwned::to_owned) + .or_else(|| thread_titles.get(&session_id).cloned()); + ctx.scan + .session_names + .extend(codex_subagent_session_name(&value, &session_id)); current_project = codex_project_context_from_value(&value, &mut project_cache); continue; } @@ -563,6 +560,7 @@ pub(crate) fn parse_codex_file( accumulated_usage: record.usage.clone(), prompt_previews: Vec::new(), last_activity_at: record.timestamp, + last_work_at: started_at, usage_lines: record .usage .as_ref() @@ -587,6 +585,11 @@ pub(crate) fn parse_codex_file( } turn.timestamp_inferred |= record.timestamp_inferred; turn.last_activity_at = record.timestamp; + // Your next prompt can land inside the open turn just before Codex + // completes it; that is the start of the next turn, not work. + if !record.is_task_complete && record.message_role.as_deref() != Some("user") { + turn.last_work_at = record.timestamp; + } if record.project.is_some() { turn.project = record.project.clone(); } @@ -688,13 +691,20 @@ pub(crate) fn parse_codex_file( consumed_usage_lines.insert(record.line_number); } let explicit_duration_ms = record.task_duration_ms; + // Without a reported duration, the turn ran until its last record + // of work, not until the completion Codex may write on resume. + let worked_until = if turn.last_work_at > turn.started_at { + turn.last_work_at.min(completed_at) + } else { + completed_at + }; let duration_ms = explicit_duration_ms - .or_else(|| codex_duration_from_turn_timestamps(turn.started_at, completed_at)); + .or_else(|| codex_duration_from_turn_timestamps(turn.started_at, worked_until)); let latency_source = explicit_duration_ms .map(|_| LatencySource::Explicit) .or_else(|| duration_ms.map(|_| LatencySource::Inferred)); let time_to_first_token_ms = record.time_to_first_token_ms; - let event = usage_event( + let mut event = usage_event( ctx.adapter, ctx.source, ctx.options, @@ -734,6 +744,7 @@ pub(crate) fn parse_codex_file( dedupe_salt: None, }, ); + event.session.title = turn.title.clone(); let mut linked_quota_lines = turn.usage_lines.clone(); linked_quota_lines.extend_from_slice(&turn.quota_lines); if record.usage.is_some() { @@ -930,7 +941,8 @@ pub(crate) fn parse_codex_file( if consumed_usage_lines.contains(&record.line_number) { continue; } - let event = usage_event( + let session_title = record.session_title; + let mut event = usage_event( ctx.adapter, ctx.source, ctx.options, @@ -966,6 +978,7 @@ pub(crate) fn parse_codex_file( dedupe_salt: None, }, ); + event.session.title = session_title; // A record's paired token_count carries the quota sample for it. let mut linked_quota_lines = vec![record.line_number]; linked_quota_lines.extend(paired_quota_lines.get(&record.line_number)); diff --git a/crates/statsai-adapters/src/codex/parse/session.rs b/crates/statsai-adapters/src/codex/parse/session.rs index 6111df6..d5f1d74 100644 --- a/crates/statsai-adapters/src/codex/parse/session.rs +++ b/crates/statsai-adapters/src/codex/parse/session.rs @@ -1,12 +1,16 @@ use super::*; -pub(crate) fn load_codex_thread_titles(root: &Path) -> HashMap { - let index_path = root.join("session_index.jsonl"); +pub(crate) const CODEX_SESSION_INDEX_FILE: &str = "session_index.jsonl"; + +/// Thread names from `session_index.jsonl`, as the provider shows them. Task +/// titles clean them further with the prompt rules; session names do not. +pub(crate) fn load_codex_thread_names(root: &Path) -> HashMap { + let index_path = root.join(CODEX_SESSION_INDEX_FILE); let Ok(file) = File::open(&index_path) else { return HashMap::new(); }; let mut reader = BufReader::new(file); - let mut titles = HashMap::new(); + let mut names = HashMap::new(); let mut line_bytes = Vec::new(); while let Ok(line_status) = read_bounded_jsonl_line(&mut reader, &mut line_bytes, MAX_JSONL_RECORD_BYTES) @@ -26,14 +30,63 @@ pub(crate) fn load_codex_thread_titles(root: &Path) -> HashMap { let Some(session_id) = value.get("id").and_then(Value::as_str) else { continue; }; - let Some(title) = value.get("thread_name").and_then(Value::as_str) else { - continue; - }; - if let Some(title) = summarize_task_text(Some(title), 90) { - titles.insert(session_id.to_string(), title); + if let Some(name) = statsai_core::provider_session_name( + value.get("thread_name").and_then(Value::as_str), + 90, + ) { + names.insert(session_id.to_string(), name); } } - titles + names +} + +pub(crate) fn codex_thread_titles_from_names( + names: &HashMap, +) -> HashMap { + names + .iter() + .filter_map(|(session_id, name)| { + summarize_task_text(Some(name), 90).map(|title| (session_id.clone(), title)) + }) + .collect() +} + +/// Codex never names its sub-agents. A spawned agent records its parent +/// thread, and a guardian review carries the parent's id as `session_id`, so +/// both borrow the parent's name, marked with the agent's nickname or kind. +/// The name is joined when the session is built, so a parent rename reaches +/// its sub-agents without re-reading their rollouts. +pub(crate) fn codex_subagent_session_name( + value: &Value, + session_id: &str, +) -> Option { + let payload = value.get("payload")?; + let subagent = payload.pointer("/source/subagent")?; + let (parent, label) = if let Some(spawn) = subagent.get("thread_spawn") { + let label = ["agent_nickname", "agent_role"] + .iter() + .find_map(|key| spawn.get(*key).and_then(Value::as_str)) + .unwrap_or("sub-agent"); + ( + spawn.get("parent_thread_id").and_then(Value::as_str)?, + label, + ) + } else { + let parent = payload + .get("session_id") + .and_then(Value::as_str) + .filter(|parent| *parent != session_id)?; + let label = subagent + .get("other") + .and_then(Value::as_str) + .unwrap_or("sub-agent"); + (parent, label) + }; + Some(statsai_core::SessionName { + local_session_id_hash: hash_text(session_id), + title: statsai_core::provider_session_name(Some(label), 40)?, + parent_local_session_id_hash: Some(hash_text(parent)), + }) } pub(crate) fn codex_project_context_from_value( diff --git a/crates/statsai-adapters/src/codex/parse/types.rs b/crates/statsai-adapters/src/codex/parse/types.rs index 2b7749e..ba0d1aa 100644 --- a/crates/statsai-adapters/src/codex/parse/types.rs +++ b/crates/statsai-adapters/src/codex/parse/types.rs @@ -108,6 +108,9 @@ pub(crate) struct ActiveCodexTurn { pub(crate) accumulated_usage: Option, pub(crate) prompt_previews: Vec, pub(crate) last_activity_at: DateTime, + /// The turn's last record before its completion. Codex writes a turn's + /// completion when the thread next runs, which can be days after the work. + pub(crate) last_work_at: DateTime, pub(crate) usage_lines: Vec, /// `token_count` lines inside the turn that carry no usage, because they /// repeat a total or pair with a `token_usage_record`. Their quota diff --git a/crates/statsai-adapters/src/codex/tests/parse/mod.rs b/crates/statsai-adapters/src/codex/tests/parse/mod.rs index 5f8906a..f7f42f0 100644 --- a/crates/statsai-adapters/src/codex/tests/parse/mod.rs +++ b/crates/statsai-adapters/src/codex/tests/parse/mod.rs @@ -178,6 +178,65 @@ fn codex_scan_candidates_ignore_auth_json_changes() { assert_eq!(first[0].cache_signature, second[0].cache_signature); } +#[test] +fn codex_session_index_is_a_scan_candidate_of_its_own() { + let dir = tempfile::tempdir().expect("tempdir"); + let sessions = dir.path().join("sessions"); + std::fs::create_dir_all(&sessions).expect("sessions"); + let session_path = sessions.join("session.jsonl"); + std::fs::write( + &session_path, + "{\"timestamp\":\"2026-05-01T00:00:00Z\",\"usage\":{\"input_tokens\":1,\"output_tokens\":2}}\n", + ) + .expect("session"); + let index_path = dir.path().join("session_index.jsonl"); + std::fs::write( + &index_path, + "{\"id\":\"thread-1\",\"thread_name\":\"One\"}\n", + ) + .expect("index one"); + + let source = SourceLocation::local_adapter( + CODEX_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let signature_of = |candidates: &[ScanCandidateFile], path: &Path| { + candidates + .iter() + .find(|candidate| candidate.cache_key == canonical_display(path)) + .map(|candidate| candidate.cache_signature.clone()) + }; + + let first = codex_scan_candidates(&source, "test-adapter").expect("first candidates"); + std::thread::sleep(std::time::Duration::from_millis(5)); + std::fs::write( + &index_path, + "{\"id\":\"thread-1\",\"thread_name\":\"One\"}\n{\"id\":\"thread-1\",\"thread_name\":\"Two\"}\n", + ) + .expect("index two"); + let second = codex_scan_candidates(&source, "test-adapter").expect("second candidates"); + + assert_eq!(first.len(), 2); + // A rename marks the index changed and leaves every rollout cached. + assert_ne!( + signature_of(&first, &index_path), + signature_of(&second, &index_path) + ); + assert_eq!( + signature_of(&first, &session_path), + signature_of(&second, &session_path) + ); + // The index is not a conversation to archive. + let archive = CodexAdapter + .archive_scan_candidates(&source) + .expect("archive candidates"); + assert_eq!(archive.len(), 1); + assert_eq!(archive[0].cache_key, canonical_display(&session_path)); +} + #[test] fn codex_scan_candidates_accept_legacy_auth_dependent_signatures() { let dir = tempfile::tempdir().expect("tempdir"); @@ -705,7 +764,8 @@ fn codex_rollout_turns_match_interleaved_records_by_session_id() { .to_rfc3339(), "2026-05-01T00:00:03+00:00" ); - assert_eq!(events[0].session.duration_seconds, Some(3)); + // Without a reported duration the turn ends at its last record of work. + assert_eq!(events[0].session.duration_seconds, Some(2)); assert_eq!(events[1].usage.total_tokens, Some(280)); assert_eq!( @@ -728,7 +788,7 @@ fn codex_rollout_turns_match_interleaved_records_by_session_id() { .to_rfc3339(), "2026-05-01T00:00:05+00:00" ); - assert_eq!(events[1].session.duration_seconds, Some(4)); + assert_eq!(events[1].session.duration_seconds, Some(3)); } #[test] diff --git a/crates/statsai-adapters/src/codex/tests/parse/session.rs b/crates/statsai-adapters/src/codex/tests/parse/session.rs index 61e7073..3850d0a 100644 --- a/crates/statsai-adapters/src/codex/tests/parse/session.rs +++ b/crates/statsai-adapters/src/codex/tests/parse/session.rs @@ -394,4 +394,149 @@ fn codex_task_spans_capture_thread_id_from_session_meta() { // account bindings all meet on one key instead of a file path. assert_eq!(scan.task_spans[0].session_id.as_deref(), Some("thread-123")); assert_eq!(scan.task_spans[0].title, "Fix parser bug"); + + // The thread name titles the session even when tasks are not collected. + let without_tasks = + scan_codex_source(&CodexAdapter, &source, &options_without_tasks()).expect("scan"); + assert!(without_tasks.task_spans.is_empty()); + assert!(!without_tasks.events.is_empty()); + assert!(without_tasks + .events + .iter() + .all(|event| event.session.title.as_deref() == Some("Fix parser bug"))); +} + +#[test] +fn codex_subagents_borrow_their_parent_thread_name() { + let dir = tempfile::tempdir().expect("tempdir"); + let codex_root = dir.path().join("codex"); + let sessions = codex_root.join("sessions"); + std::fs::create_dir_all(&sessions).expect("sessions"); + std::fs::write( + codex_root.join("session_index.jsonl"), + "{\"id\":\"parent-1\",\"thread_name\":\"Fix parser bug\"}\n", + ) + .expect("session index"); + for (index, (file, meta)) in [ + ( + "spawned.jsonl", + r#"{"timestamp":"2026-06-01T08:00:00Z","type":"session_meta","payload":{"id":"child-spawned","source":{"subagent":{"thread_spawn":{"parent_thread_id":"parent-1","depth":1,"agent_nickname":"Popper","agent_role":null}}}}}"#, + ), + ( + "guardian.jsonl", + r#"{"timestamp":"2026-06-01T08:00:00Z","type":"session_meta","payload":{"id":"child-guardian","session_id":"parent-1","source":{"subagent":{"other":"guardian"}}}}"#, + ), + ( + "orphan.jsonl", + r#"{"timestamp":"2026-06-01T08:00:00Z","type":"session_meta","payload":{"id":"child-orphan","source":{"subagent":{"other":"guardian"}}}}"#, + ), + ] + .into_iter() + .enumerate() + { + // Distinct usage per rollout, or the path-independent dedupe keeps one. + let input = 60 + index; + let total = 120 + index; + let usage = format!( + r#"{{"timestamp":"2026-06-01T08:00:03Z","type":"event_msg","payload":{{"type":"token_count","info":{{"last_token_usage":{{"input_tokens":{input},"cached_input_tokens":20,"output_tokens":30,"reasoning_output_tokens":10,"total_tokens":{total}}},"total_token_usage":{{"input_tokens":{input},"cached_input_tokens":20,"output_tokens":30,"reasoning_output_tokens":10,"total_tokens":{total}}}}}}}}}"# + ); + let mut handle = File::create(sessions.join(file)).expect("rollout"); + writeln!(handle, "{meta}").expect("meta"); + writeln!( + handle, + r#"{{"timestamp":"2026-06-01T08:00:00Z","type":"turn_context","payload":{{"model":"gpt-5"}}}}"# + ) + .expect("context"); + writeln!(handle, "{usage}").expect("usage"); + } + + let source = SourceLocation::local_adapter( + CODEX_PROVIDER, + "test", + "0", + &codex_root, + LocationOrigin::Configured, + ); + let scan = scan_codex_source(&CodexAdapter, &source, &options_without_tasks()).expect("scan"); + // The parent's name is joined when the session is built, so a rename of + // the parent reaches its sub-agents; here only the link is reported. + let link_for = |raw: &str| { + let hash = statsai_core::hash_text(raw); + scan.session_names + .iter() + .find(|name| name.local_session_id_hash == hash) + .cloned() + }; + let parent = Some(statsai_core::hash_text("parent-1")); + assert_eq!( + link_for("child-spawned").map(|name| (name.title, name.parent_local_session_id_hash)), + Some(("Popper".to_string(), parent.clone())) + ); + assert_eq!( + link_for("child-guardian").map(|name| (name.title, name.parent_local_session_id_hash)), + Some(("guardian".to_string(), parent)) + ); + assert_eq!(link_for("child-orphan"), None); +} + +#[test] +fn codex_turns_without_a_reported_duration_end_at_their_last_work() { + let dir = tempfile::tempdir().expect("tempdir"); + let codex_root = dir.path().join("codex"); + let sessions = codex_root.join("sessions"); + std::fs::create_dir_all(&sessions).expect("sessions"); + let mut file = File::create(sessions.join("resumed.jsonl")).expect("rollout"); + for line in [ + r#"{"timestamp":"2026-06-01T08:00:00Z","type":"session_meta","payload":{"id":"resumed-thread"}}"#, + r#"{"timestamp":"2026-06-01T08:00:00Z","type":"turn_context","payload":{"model":"gpt-5"}}"#, + r#"{"timestamp":"2026-06-01T08:00:01Z","type":"event_msg","payload":{"type":"task_started","started_at":"2026-06-01T08:00:01Z"}}"#, + r#"{"timestamp":"2026-06-01T08:02:01Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":60,"cached_input_tokens":20,"output_tokens":30,"reasoning_output_tokens":10,"total_tokens":120},"total_token_usage":{"input_tokens":60,"cached_input_tokens":20,"output_tokens":30,"reasoning_output_tokens":10,"total_tokens":120}}}}"#, + // The next prompt and the completion are written three days later, + // when the thread runs again, with no duration. + r#"{"timestamp":"2026-06-04T09:00:00Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"Next step"}]}}"#, + r#"{"timestamp":"2026-06-04T09:00:00Z","type":"event_msg","payload":{"type":"task_complete","completed_at":"2026-06-04T09:00:00Z"}}"#, + ] { + writeln!(file, "{line}").expect("write line"); + } + + let source = SourceLocation::local_adapter( + CODEX_PROVIDER, + "test", + "0", + &codex_root, + LocationOrigin::Configured, + ); + let scan = scan_codex_source(&CodexAdapter, &source, &options_without_tasks()).expect("scan"); + assert_eq!(scan.events.len(), 1); + assert_eq!(scan.events[0].session.duration_seconds, Some(120)); +} + +#[test] +fn codex_session_index_names_are_reported_on_every_scan() { + let dir = tempfile::tempdir().expect("tempdir"); + let codex_root = dir.path().join("codex"); + std::fs::create_dir_all(codex_root.join("sessions")).expect("sessions"); + std::fs::write( + codex_root.join("session_index.jsonl"), + "{\"id\":\"thread-1\",\"thread_name\":\"Design tool/plugin\"}\n", + ) + .expect("session index"); + + let source = SourceLocation::local_adapter( + CODEX_PROVIDER, + "test", + "0", + &codex_root, + LocationOrigin::Configured, + ); + let scan = scan_codex_source(&CodexAdapter, &source, &options_without_tasks()).expect("scan"); + // No rollout changed, yet the index name is reported, uncleaned. + assert_eq!( + scan.session_names, + vec![statsai_core::SessionName { + local_session_id_hash: statsai_core::hash_text("thread-1"), + title: "Design tool/plugin".to_string(), + parent_local_session_id_hash: None, + }] + ); } diff --git a/crates/statsai-adapters/src/event.rs b/crates/statsai-adapters/src/event.rs index c827aa7..f38a812 100644 --- a/crates/statsai-adapters/src/event.rs +++ b/crates/statsai-adapters/src/event.rs @@ -134,6 +134,7 @@ pub(crate) fn merge_adapter_scan( push_deduped(target, index, event, DuplicateSelection::KeepFirst); } target.summaries.append(&mut source.summaries); + target.session_names.append(&mut source.session_names); target.task_spans.append(&mut source.task_spans); target .quota_observations @@ -225,6 +226,32 @@ pub(crate) struct ProviderEventParts<'a> { pub(crate) dedupe_salt: Option, } +pub(crate) fn message_count_runtime( + user_messages: u64, + assistant_messages: u64, + developer_messages: u64, +) -> Option { + let total = user_messages + .saturating_add(assistant_messages) + .saturating_add(developer_messages); + if total == 0 { + return None; + } + Some(RuntimeInfo { + runtime_name: None, + host_id: None, + latency_ms: None, + latency_source: None, + time_to_first_token_ms: None, + prompt_eval_duration_ms: None, + eval_duration_ms: None, + total_messages: Some(total), + user_messages: (user_messages > 0).then_some(user_messages), + assistant_messages: (assistant_messages > 0).then_some(assistant_messages), + developer_messages: (developer_messages > 0).then_some(developer_messages), + }) +} + pub(crate) fn usage_event( adapter: &A, source: &SourceLocation, @@ -355,6 +382,7 @@ pub(crate) fn usage_event( started_at: session_started_at, ended_at: parts.session_ended_at, duration_seconds: parts.duration_seconds, + turn_started_at: None, }, model: parts.model, runtime: parts.runtime, diff --git a/crates/statsai-adapters/src/lib.rs b/crates/statsai-adapters/src/lib.rs index 5924efb..01d4962 100644 --- a/crates/statsai-adapters/src/lib.rs +++ b/crates/statsai-adapters/src/lib.rs @@ -54,10 +54,10 @@ pub(crate) use cache::{ scan_candidate_with_compatible_dependencies, ScanCacheNamespaces, }; pub(crate) use event::{ - infer_missing_output, merge_adapter_scan, metadata_only_privacy, metadata_summary, - metric_from_samples, metric_single_sample, push_deduped, subtract_usage_counts, - sum_usage_counts, usage_event, usd_to_micro_usd, DuplicateSelection, EventDedupIndex, - EventDeduplication, MetadataSummaryParts, ProviderEventParts, + infer_missing_output, merge_adapter_scan, message_count_runtime, metadata_only_privacy, + metadata_summary, metric_from_samples, metric_single_sample, push_deduped, + subtract_usage_counts, sum_usage_counts, usage_event, usd_to_micro_usd, DuplicateSelection, + EventDedupIndex, EventDeduplication, MetadataSummaryParts, ProviderEventParts, }; pub(crate) use json::{ file_modified_timestamp, number_at_any, read_bounded_jsonl_line, read_json_file, @@ -168,6 +168,8 @@ pub struct AdapterScan { pub events: Vec, pub summaries: Vec, pub task_spans: Vec, + /// Session names read apart from usage events; see [`SessionName`]. + pub session_names: Vec, pub quota_observations: Vec, pub activity_invocations: Vec, pub activity_coverage: Vec, diff --git a/crates/statsai-adapters/src/opencode/mod.rs b/crates/statsai-adapters/src/opencode/mod.rs index 6dda9e0..37548c0 100644 --- a/crates/statsai-adapters/src/opencode/mod.rs +++ b/crates/statsai-adapters/src/opencode/mod.rs @@ -62,6 +62,7 @@ pub(crate) fn scan_opencode_source( let connection = open_sqlite_readonly(&db_path)?; let todos_by_session = load_opencode_todos(&connection)?; let recovered_session_models = load_opencode_session_models(&connection)?; + let message_role_counts = load_opencode_message_role_counts(&connection)?; let reconstructed_session_ids = recovered_session_models .iter() .filter_map(|(session_id, summary)| { @@ -218,6 +219,9 @@ pub(crate) fn scan_opencode_source( }) .filter(|cost| cost.estimated_micro_usd().is_some()); let priced_from_messages = message_cost.is_some(); + let runtime = message_role_counts.get(&session_id).and_then(|counts| { + message_count_runtime(counts.user, counts.assistant, counts.developer) + }); let mut event = usage_event( adapter, source, @@ -229,7 +233,7 @@ pub(crate) fn scan_opencode_source( duration_seconds, model, usage, - runtime: None, + runtime, session_raw: session_id, project, event_kind: "opencode_session_usage", @@ -499,6 +503,40 @@ pub(crate) fn scan_opencode_source( Ok(scan) } +#[derive(Debug, Clone, Copy, Default)] +struct OpenCodeMessageCounts { + user: u64, + assistant: u64, + developer: u64, +} + +fn load_opencode_message_role_counts( + connection: &Connection, +) -> Result> { + let mut statement = + match connection.prepare("SELECT session_id, json_extract(data, '$.role') FROM message") { + Ok(statement) => statement, + Err(error) if error.to_string().contains("no such table: message") => { + return Ok(HashMap::new()); + } + Err(error) => return Err(error.into()), + }; + let mut rows = statement.query([])?; + let mut counts = HashMap::::new(); + while let Some(row) = rows.next()? { + let session_id: String = row.get(0)?; + let role: Option = row.get(1).ok().flatten(); + let entry = counts.entry(session_id).or_default(); + match role.as_deref() { + Some("user") => entry.user = entry.user.saturating_add(1), + Some("assistant") => entry.assistant = entry.assistant.saturating_add(1), + Some("developer") => entry.developer = entry.developer.saturating_add(1), + _ => {} + } + } + Ok(counts) +} + pub(crate) fn load_opencode_session_models( connection: &Connection, ) -> Result> { @@ -753,6 +791,7 @@ pub(crate) fn emit_opencode_message_events( let mut rows = statement.query([])?; let mut reconstructed_usage = HashMap::::new(); let mut session_models = HashMap::::new(); + let mut pending_user_messages = HashMap::::new(); while let Some(row) = rows.next()? { let session_id: String = row.get(1)?; if !ctx.reconstructed_session_ids.contains(&session_id) { @@ -776,7 +815,11 @@ pub(crate) fn emit_opencode_message_events( session_models.insert(session_id.clone(), model); } let usage = opencode_message_usage_counts(&value); + let role = value.get("role").and_then(Value::as_str); if usage.computed_total() == 0 { + if role == Some("user") { + *pending_user_messages.entry(session_id).or_default() += 1; + } ctx.scan.diagnostics.skipped_zero_events += 1; continue; } @@ -785,6 +828,10 @@ pub(crate) fn emit_opencode_message_events( ctx.scan.diagnostics.model_fallbacks += 1; continue; }; + let pending_users = pending_user_messages.remove(&session_id).unwrap_or(0); + let user_messages = pending_users.saturating_add(u64::from(role == Some("user"))); + let assistant_messages = u64::from(role != Some("user") && role != Some("developer")); + let developer_messages = u64::from(role == Some("developer")); let started_at = value .pointer("/time/created") .and_then(value_as_u64) @@ -817,7 +864,11 @@ pub(crate) fn emit_opencode_message_events( duration_seconds, model: Some(model), usage, - runtime: None, + runtime: message_count_runtime( + user_messages, + assistant_messages, + developer_messages, + ), session_raw: session_id.clone(), project, event_kind: "opencode_message_usage", diff --git a/crates/statsai-core/src/tasks/normalize/mod.rs b/crates/statsai-core/src/tasks/normalize/mod.rs index 9579ac2..7f6e6b6 100644 --- a/crates/statsai-core/src/tasks/normalize/mod.rs +++ b/crates/statsai-core/src/tasks/normalize/mod.rs @@ -49,6 +49,24 @@ pub fn summarize_task_text(value: Option<&str>, width: usize) -> Option truncate_task_text(clean_task_text(value?)?, width) } +/// Normalizes a session name the provider assigned: control characters and +/// runs of whitespace collapse to single spaces and the result is truncated. +/// +/// [`summarize_task_text`] cleans prompts, and its line filters drop a name +/// like "Design tool/plugin" as a file reference. +#[must_use] +pub fn provider_session_name(value: Option<&str>, width: usize) -> Option { + let compact = value? + .split(|character: char| character.is_whitespace() || character.is_control()) + .filter(|part| !part.is_empty()) + .collect::>() + .join(" "); + if compact.is_empty() { + return None; + } + truncate_task_text(compact, width) +} + #[must_use] pub fn choose_best_task_title<'a>( primary: Option<&'a str>, diff --git a/crates/statsai-core/src/tasks/titles/mod.rs b/crates/statsai-core/src/tasks/titles/mod.rs index ff399c2..8341605 100644 --- a/crates/statsai-core/src/tasks/titles/mod.rs +++ b/crates/statsai-core/src/tasks/titles/mod.rs @@ -96,6 +96,39 @@ pub fn task_title_signal_score(value: Option<&str>) -> i32 { score } +/// Whether a session name the provider itself assigned is unfit to show. +/// +/// Provider names are shown as the provider shows them, so "Review +/// uncommitted changes" or "Say hi" stay. The genericness rules in +/// [`task_title_is_generic`] rank prompt-derived task titles; here only a +/// placeholder or text that looks like a secret or a locator is dropped. +#[must_use] +pub fn provider_session_title_is_unusable(value: Option<&str>) -> bool { + let Some(name) = super::normalize::provider_session_name(value, usize::MAX) else { + return true; + }; + looks_like_sensitive_locator_dump(&name) + || looks_like_provider_placeholder_title(&name) + || looks_like_session_placeholder_name(&name) +} + +/// Default names a tool gives a session before anything names it, such as +/// OpenCode's "New session - 2026-04-30T16:41:41.413Z". +const SESSION_PLACEHOLDER_NAMES: &[&str] = &[ + "new session", + "new chat", + "new conversation", + "new thread", + "untitled", + "untitled session", +]; + +fn looks_like_session_placeholder_name(name: &str) -> bool { + let lowercase = name.to_ascii_lowercase(); + let head = lowercase.split(" - ").next().unwrap_or_default().trim(); + SESSION_PLACEHOLDER_NAMES.contains(&head) +} + #[must_use] pub fn task_title_is_generic(value: Option<&str>) -> bool { let Some(raw) = value else { diff --git a/crates/statsai-core/src/tests/support.rs b/crates/statsai-core/src/tests/support.rs index c183d4e..ce65aee 100644 --- a/crates/statsai-core/src/tests/support.rs +++ b/crates/statsai-core/src/tests/support.rs @@ -43,6 +43,7 @@ pub(super) fn test_event( started_at, ended_at: None, duration_seconds: None, + turn_started_at: None, }, model: None, usage: UsageCounts { diff --git a/crates/statsai-core/src/tests/types.rs b/crates/statsai-core/src/tests/types.rs index d5103dd..e852347 100644 --- a/crates/statsai-core/src/tests/types.rs +++ b/crates/statsai-core/src/tests/types.rs @@ -1,5 +1,6 @@ use super::support::*; use super::*; +use chrono::{DateTime, Utc}; use std::path::Path; #[test] fn source_ids_are_stable_for_same_input() { @@ -58,6 +59,182 @@ fn schema_types_serialize() { let schema = schemars::schema_for!(UsageSummary); let json = serde_json::to_value(schema).expect("summary schema should serialize"); assert!(json.get("title").is_some()); + + let schema = schemars::schema_for!(SessionRollupV1); + let json = serde_json::to_value(schema).expect("session rollup schema should serialize"); + assert!(json.get("title").is_some()); +} + +#[test] +fn session_rollup_uses_snake_case_contract() { + let started = DateTime::parse_from_rfc3339("2026-05-01T00:00:00Z") + .expect("start") + .with_timezone(&Utc); + let rollup = SessionRollupV1 { + schema_version: SESSION_ROLLUP_SCHEMA_VERSION.to_string(), + session_id: "session_abc".to_string(), + device_id: "device".to_string(), + provider: "codex".to_string(), + source_id: SourceId("src".to_string()), + provider_account_id: None, + started_at: started, + ended_at: started, + duration_seconds: Some(0), + active_seconds: None, + usage: UsageCounts::default(), + requests: 1, + cost: CostInfo { + currency: "USD".to_string(), + estimated_api_equivalent_usd: None, + provider_reported_usd: None, + estimated_api_equivalent_micro_usd: Some(10), + provider_reported_micro_usd: None, + pricing_source: None, + pricing_version: None, + confidence: Confidence::Medium, + }, + models: Vec::new(), + primary_model: Some("gpt-5".to_string()), + total_messages: Some(2), + user_messages: Some(1), + assistant_messages: Some(1), + developer_messages: None, + project: None, + title: Some("Fix parser".to_string()), + title_source: Some(SessionTitleSource::TaskSpan), + updated_at: started, + }; + let json = serde_json::to_value(&rollup).expect("serialize"); + for field in [ + "schema_version", + "session_id", + "device_id", + "provider", + "source_id", + "provider_account_id", + "started_at", + "ended_at", + "duration_seconds", + "active_seconds", + "usage", + "requests", + "cost", + "models", + "primary_model", + "total_messages", + "user_messages", + "assistant_messages", + "developer_messages", + "project", + "title", + "title_source", + "updated_at", + ] { + assert!(json.get(field).is_some(), "missing {field}"); + } + assert_eq!(json["schema_version"], "session_rollup.v1"); + assert_eq!(json["title_source"], "task_span"); + assert_eq!(json["requests"], 1); + assert_eq!(json["duration_seconds"], 0); + assert!(json["usage"].get("cache_creation_5m_tokens").is_none()); + assert!(json["usage"].get("cache_creation_1h_tokens").is_none()); + assert!(json["cost"].get("currency").is_none()); + assert!(json["cost"].get("pricing_source").is_none()); + assert!(json["cost"].get("confidence").is_none()); + let restored: SessionRollupV1 = serde_json::from_value(json).expect("deserialize"); + assert_eq!(restored, rollup); +} + +#[test] +fn session_rollup_unknown_duration_serializes_as_null() { + let started = DateTime::parse_from_rfc3339("2026-05-01T00:00:00Z") + .expect("start") + .with_timezone(&Utc); + let mut rollup = SessionRollupV1 { + schema_version: SESSION_ROLLUP_SCHEMA_VERSION.to_string(), + session_id: "session_abc".to_string(), + device_id: "device".to_string(), + provider: "codex".to_string(), + source_id: SourceId("src".to_string()), + provider_account_id: None, + started_at: started, + ended_at: started, + duration_seconds: None, + active_seconds: None, + usage: UsageCounts::default(), + requests: 0, + cost: CostInfo { + currency: "USD".to_string(), + estimated_api_equivalent_usd: None, + provider_reported_usd: None, + estimated_api_equivalent_micro_usd: None, + provider_reported_micro_usd: None, + pricing_source: None, + pricing_version: None, + confidence: Confidence::Medium, + }, + models: Vec::new(), + primary_model: None, + total_messages: None, + user_messages: None, + assistant_messages: None, + developer_messages: None, + project: None, + title: None, + title_source: None, + updated_at: started, + }; + let json = serde_json::to_value(&rollup).expect("serialize"); + assert!(json["duration_seconds"].is_null()); + let restored: SessionRollupV1 = serde_json::from_value(json).expect("deserialize"); + assert_eq!(restored, rollup); + rollup.duration_seconds = Some(0); + let json = serde_json::to_value(&rollup).expect("serialize measured"); + assert_eq!(json["duration_seconds"], 0); +} + +#[test] +fn sync_batch_omits_empty_sessions_and_absent_session_retirement() { + let batch = SyncBatch { + schema_version: SYNC_BATCH_V6_SCHEMA_VERSION.to_string(), + batch_id: "batch-sessions".to_string(), + device_id: "device-1".to_string(), + sources: Vec::new(), + accounts: Vec::new(), + source_account_assignments: Vec::new(), + subscriptions: Vec::new(), + events: Vec::new(), + summaries: Vec::new(), + task_buckets: Vec::new(), + task_verifications: Vec::new(), + code_change_metrics: Vec::new(), + quota_cycle_contributions: Vec::new(), + account_plan_observations: Vec::new(), + account_evidence_summaries: Vec::new(), + activity_rollups: Vec::new(), + activity_coverage: Vec::new(), + sessions: Vec::new(), + authoritative_snapshot: Some(SyncAuthoritativeSnapshot::default()), + created_at: DateTime::parse_from_rfc3339("2026-05-31T10:00:00Z") + .expect("timestamp") + .with_timezone(&Utc), + }; + let json = serde_json::to_value(&batch).expect("serialize"); + assert!(json.get("sessions").is_none()); + assert!(json["authoritative_snapshot"] + .get("session_rollup_ids") + .is_none()); + + let mut retiring = batch; + retiring.authoritative_snapshot = Some(SyncAuthoritativeSnapshot { + session_rollup_ids: Some(Vec::new()), + ..SyncAuthoritativeSnapshot::default() + }); + let json = serde_json::to_value(&retiring).expect("serialize retirement"); + assert_eq!( + json["authoritative_snapshot"]["session_rollup_ids"], + serde_json::json!([]) + ); } #[test] @@ -95,6 +272,7 @@ fn sync_ack_v1_omits_zero_task_counters() { account_evidence_summaries: 0, activity_rollups: 0, activity_coverage: 0, + sessions: 0, }, duplicates: SyncEntityCounts { sources: 0, @@ -111,6 +289,7 @@ fn sync_ack_v1_omits_zero_task_counters() { account_evidence_summaries: 0, activity_rollups: 0, activity_coverage: 0, + sessions: 0, }, rejected: Vec::new(), }; @@ -123,6 +302,8 @@ fn sync_ack_v1_omits_zero_task_counters() { assert!(json["duplicates"].get("task_buckets").is_none()); assert!(json["duplicates"].get("task_verifications").is_none()); assert!(json["duplicates"].get("code_change_metrics").is_none()); + assert!(json["accepted"].get("sessions").is_none()); + assert!(json["duplicates"].get("sessions").is_none()); } #[test] @@ -145,6 +326,7 @@ fn sync_ack_v3_keeps_nonzero_task_and_code_change_counters() { account_evidence_summaries: 0, activity_rollups: 0, activity_coverage: 0, + sessions: 0, }, duplicates: SyncEntityCounts { sources: 0, @@ -161,6 +343,7 @@ fn sync_ack_v3_keeps_nonzero_task_and_code_change_counters() { account_evidence_summaries: 0, activity_rollups: 0, activity_coverage: 0, + sessions: 0, }, rejected: Vec::new(), }; @@ -195,6 +378,7 @@ fn sync_ack_v4_keeps_nonzero_quota_cycle_counters() { account_evidence_summaries: 0, activity_rollups: 0, activity_coverage: 0, + sessions: 0, }, duplicates: SyncEntityCounts { sources: 0, @@ -211,6 +395,7 @@ fn sync_ack_v4_keeps_nonzero_quota_cycle_counters() { account_evidence_summaries: 0, activity_rollups: 0, activity_coverage: 0, + sessions: 0, }, rejected: Vec::new(), }; @@ -428,3 +613,303 @@ fn sanitize_summary_for_sync_marks_project_path_labels_as_file_paths() { ); assert!(sanitized.privacy.contains_file_paths); } + +fn sample_session_rollup(started: DateTime) -> SessionRollupV1 { + SessionRollupV1 { + schema_version: SESSION_ROLLUP_SCHEMA_VERSION.to_string(), + session_id: "session_abc".to_string(), + device_id: "device".to_string(), + provider: "codex".to_string(), + source_id: SourceId("src".to_string()), + provider_account_id: None, + started_at: started, + ended_at: started, + duration_seconds: Some(0), + active_seconds: None, + usage: UsageCounts::default(), + requests: 1, + cost: CostInfo { + currency: "USD".to_string(), + estimated_api_equivalent_usd: None, + provider_reported_usd: None, + estimated_api_equivalent_micro_usd: Some(10), + provider_reported_micro_usd: None, + pricing_source: None, + pricing_version: None, + confidence: Confidence::Medium, + }, + models: Vec::new(), + primary_model: Some("gpt-5".to_string()), + total_messages: None, + user_messages: None, + assistant_messages: None, + developer_messages: None, + project: None, + title: Some("Fix parser".to_string()), + title_source: Some(SessionTitleSource::TaskSpan), + updated_at: started, + } +} + +#[test] +fn sanitize_session_rollup_repairs_values_ingest_rejects() { + let started = DateTime::parse_from_rfc3339("2026-05-01T00:00:00Z") + .expect("start") + .with_timezone(&Utc); + let mut inverted = sample_session_rollup(started + chrono::Duration::hours(2)); + inverted.ended_at = started; + inverted.duration_seconds = Some(7_200); + let sanitized = sanitize_session_rollup_for_sync(inverted); + assert_eq!(sanitized.ended_at, sanitized.started_at); + assert_eq!(sanitized.duration_seconds, Some(0)); + + let mut provider_named = sample_session_rollup(started); + provider_named.title = Some("Review uncommitted changes".to_string()); + provider_named.title_source = Some(SessionTitleSource::Event); + let sanitized = sanitize_session_rollup_for_sync(provider_named); + assert_eq!( + sanitized.title.as_deref(), + Some("Review uncommitted changes") + ); + let mut slashed = sample_session_rollup(started); + slashed.title = Some("Claude Design tool/plugin".to_string()); + slashed.title_source = Some(SessionTitleSource::Event); + let sanitized = sanitize_session_rollup_for_sync(slashed); + assert_eq!( + sanitized.title.as_deref(), + Some("Claude Design tool/plugin") + ); + let mut prompt_derived = sample_session_rollup(started); + prompt_derived.title = Some("Review uncommitted changes".to_string()); + prompt_derived.title_source = Some(SessionTitleSource::TaskSpan); + let sanitized = sanitize_session_rollup_for_sync(prompt_derived); + assert_eq!(sanitized.title, None); + assert_eq!(sanitized.title_source, None); + + let mut unknown = sample_session_rollup(started); + unknown.duration_seconds = None; + let sanitized = sanitize_session_rollup_for_sync(unknown); + assert_eq!(sanitized.duration_seconds, None); + + let mut future = sample_session_rollup(started); + future.started_at = Utc::now() + chrono::Duration::hours(48); + future.ended_at = future.started_at; + let sanitized = sanitize_session_rollup_for_sync(future); + assert_eq!(sanitized.started_at, started); + assert_eq!(sanitized.ended_at, started); + assert!(sanitized.updated_at <= Utc::now() + chrono::Duration::hours(24)); + + let mut titled = sample_session_rollup(started); + let mut title = "A".repeat(300); + title.insert(10, '\n'); + titled.title = Some(title); + titled.title_source = Some(SessionTitleSource::Event); + let sanitized = sanitize_session_rollup_for_sync(titled); + let title = sanitized.title.expect("repaired title"); + assert!(!title.chars().any(|ch| ch.is_control())); + assert!(title.encode_utf16().count() <= 256); + assert_eq!(sanitized.title_source, Some(SessionTitleSource::Event)); + + let mut generic = sample_session_rollup(started); + generic.title = Some("work item".to_string()); + generic.title_source = Some(SessionTitleSource::Archive); + let sanitized = sanitize_session_rollup_for_sync(generic); + assert!(sanitized.title.is_none()); + assert!(sanitized.title_source.is_none()); + + let mut counted = sample_session_rollup(started); + counted.usage.input_tokens = Some(1_000_000_000_005); + counted.usage.requests = Some(1_000_000_005); + counted.requests = 1_000_000_005; + counted.cost.estimated_api_equivalent_micro_usd = Some(-25); + counted.total_messages = Some(u64::MAX); + counted.models = (0..65) + .map(|index| SummaryModelUsage { + model: ModelInfo { + normalized_name: Some(format!("m{index}")), + ..ModelInfo::default() + }, + usage: UsageCounts { + total_tokens: Some(index), + ..UsageCounts::default() + }, + cost: CostInfo { + currency: "USD".to_string(), + estimated_api_equivalent_usd: None, + provider_reported_usd: None, + estimated_api_equivalent_micro_usd: None, + provider_reported_micro_usd: None, + pricing_source: None, + pricing_version: None, + confidence: Confidence::Medium, + }, + metrics: None, + }) + .collect(); + let sanitized = sanitize_session_rollup_for_sync(counted); + assert_eq!(sanitized.usage.input_tokens, Some(1_000_000_000_000)); + assert_eq!(sanitized.usage.requests, Some(1_000_000_000)); + assert_eq!(sanitized.requests, 1_000_000_000); + assert_eq!(sanitized.cost.estimated_api_equivalent_micro_usd, Some(0)); + assert_eq!(sanitized.total_messages, Some((1_u64 << 53) - 1)); + assert_eq!(sanitized.models.len(), 64); + assert!(sanitized + .models + .iter() + .all(|model| model.model.normalized_name.as_deref() != Some("m0"))); +} + +#[test] +fn sanitized_session_rollup_uses_ingest_allowlist() { + let started = DateTime::parse_from_rfc3339("2026-05-01T00:00:00Z") + .expect("start") + .with_timezone(&Utc); + let mut rollup = sample_session_rollup(started); + rollup.usage.cache_creation_5m_tokens = Some(4); + rollup.usage.cache_creation_1h_tokens = Some(8); + rollup.usage.cache_creation_tokens = Some(12); + rollup.project = Some(ProjectInfo { + project_id: "project_path_only".to_string(), + project_label: Some("Scratch".to_string()), + repo_remote_hash: Some("remote".to_string()), + repo_label: Some("statsai".to_string()), + branch_hash: Some("branch".to_string()), + branch_label: Some("main".to_string()), + path_hash: Some("path".to_string()), + path_label: Some("/Users/example/Scratch".to_string()), + }); + rollup.models = vec![SummaryModelUsage { + model: ModelInfo { + name: Some("GPT-5".to_string()), + normalized_name: Some("gpt-5".to_string()), + provider_model_id: Some("gpt-5".to_string()), + speed: Some("standard".to_string()), + reasoning_level: Some(ReasoningLevel::High), + reasoning_level_raw: Some("high".to_string()), + }, + usage: UsageCounts { + input_tokens: Some(1), + cache_creation_5m_tokens: Some(2), + ..UsageCounts::default() + }, + cost: rollup.cost.clone(), + metrics: Some(SummaryModelMetrics { + generated_tps: Some(SummaryMetricTotals { + samples: 1, + sum: 3.0, + }), + }), + }]; + let json = serde_json::to_value(sanitize_session_rollup_for_sync(rollup)).expect("json"); + assert_allowlist( + &json, + &[ + "schema_version", + "session_id", + "device_id", + "provider", + "source_id", + "provider_account_id", + "started_at", + "ended_at", + "duration_seconds", + "active_seconds", + "usage", + "requests", + "cost", + "models", + "primary_model", + "total_messages", + "user_messages", + "assistant_messages", + "developer_messages", + "project", + "title", + "title_source", + "updated_at", + ], + ); + assert_allowlist( + &json["usage"], + &[ + "total_tokens", + "input_tokens", + "output_tokens", + "cache_creation_tokens", + "cache_read_tokens", + "reasoning_tokens", + "local_prompt_eval_tokens", + "local_eval_tokens", + "requests", + ], + ); + assert!(json["usage"].get("cache_creation_5m_tokens").is_none()); + assert!(json["usage"].get("cache_creation_1h_tokens").is_none()); + assert_allowlist( + &json["cost"], + &[ + "provider_reported_usd", + "estimated_api_equivalent_usd", + "provider_reported_micro_usd", + "estimated_api_equivalent_micro_usd", + ], + ); + assert!(json["cost"].get("currency").is_none()); + assert!(json["cost"].get("confidence").is_none()); + assert_allowlist( + &json["project"], + &[ + "project_id", + "project_label", + "repo_remote_hash", + "repo_label", + "branch_hash", + "branch_label", + "path_hash", + "path_label", + ], + ); + let model = &json["models"][0]; + assert_allowlist(model, &["model", "usage", "cost", "metrics"]); + assert_allowlist( + &model["model"], + &[ + "normalized_name", + "name", + "provider_model_id", + "speed", + "reasoning_level", + "reasoning_level_raw", + ], + ); + assert_allowlist( + &model["usage"], + &[ + "total_tokens", + "input_tokens", + "output_tokens", + "cache_creation_tokens", + "cache_read_tokens", + "reasoning_tokens", + "local_prompt_eval_tokens", + "local_eval_tokens", + "requests", + ], + ); + assert!(model["usage"].get("cache_creation_5m_tokens").is_none()); + assert_allowlist(&model["metrics"], &["generated_tps"]); + assert_allowlist( + &model["metrics"]["generated_tps"], + &["samples", "avg", "min", "max", "p50", "p95", "sum"], + ); +} + +fn assert_allowlist(value: &serde_json::Value, allowed: &[&str]) { + let object = value.as_object().expect("object"); + for key in object.keys() { + assert!( + allowed.contains(&key.as_str()), + "unexpected key {key} in {value}" + ); + } +} diff --git a/crates/statsai-core/src/types/event.rs b/crates/statsai-core/src/types/event.rs index 3f66831..acc7695 100644 --- a/crates/statsai-core/src/types/event.rs +++ b/crates/statsai-core/src/types/event.rs @@ -33,6 +33,11 @@ pub struct SessionInfo { pub started_at: DateTime, pub ended_at: Option>, pub duration_seconds: Option, + /// When the prompt that started this event's turn was sent, for providers + /// that record one timestamp per message. The session's active time runs + /// from here to the event. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub turn_started_at: Option>, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] diff --git a/crates/statsai-core/src/types/mod.rs b/crates/statsai-core/src/types/mod.rs index 952a554..328f5fb 100644 --- a/crates/statsai-core/src/types/mod.rs +++ b/crates/statsai-core/src/types/mod.rs @@ -1,6 +1,7 @@ mod account; mod activity; mod event; +mod session; mod source; mod subscription; mod summary; @@ -9,6 +10,7 @@ mod sync; pub use account::*; pub use activity::*; pub use event::*; +pub use session::*; pub use source::*; pub use subscription::*; pub use summary::*; diff --git a/crates/statsai-core/src/types/session.rs b/crates/statsai-core/src/types/session.rs new file mode 100644 index 0000000..40a2be1 --- /dev/null +++ b/crates/statsai-core/src/types/session.rs @@ -0,0 +1,279 @@ +use super::event::{Confidence, CostInfo, ModelInfo, ProjectInfo, UsageCounts}; +use super::summary::{SummaryModelMetrics, SummaryModelUsage}; +use crate::ids::{ProviderAccountId, SourceId}; +use chrono::{DateTime, Utc}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +pub const SESSION_ROLLUP_SCHEMA_VERSION: &str = "session_rollup.v1"; + +/// Where a session title was resolved from. Event titles win over task spans, +/// which win over archived conversations. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum SessionTitleSource { + Event, + TaskSpan, + Archive, +} + +/// A name the provider holds for a session, read apart from its usage events. +/// +/// Names live in files the scan cache does not tie to usage: Codex keeps them +/// in `session_index.jsonl`, and Claude Code writes a renamed title into the +/// transcript that was resumed, while the usage stays in an unchanged one. A +/// rename therefore reaches the session without re-parsing its events. +/// +/// A sub-agent the provider never names borrows its parent's name instead: +/// `parent_local_session_id_hash` points at the parent, `title` is the +/// sub-agent's own label, and the session reads "{parent name} · {label}" +/// with whatever name the parent holds at the time. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SessionName { + pub local_session_id_hash: String, + pub title: String, + pub parent_local_session_id_hash: Option, +} + +/// One local session, aggregated from the usage events that share its hashed id. +/// +/// `session_id` is the hashed `session_…` identity already stored on each event. +/// Raw provider session ids never appear on this record. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct SessionRollupV1 { + pub schema_version: String, + pub session_id: String, + pub device_id: String, + pub provider: String, + pub source_id: SourceId, + pub provider_account_id: Option, + pub started_at: DateTime, + pub ended_at: DateTime, + /// Wall-clock span from the first event to the last, idle time included. + /// `None` when the session recorded only a start. Unknown durations go out + /// as null; ingest stores them without an end so they stay out of medians. + pub duration_seconds: Option, + /// Time the agent was working: the union of its turns, each from the prompt + /// until the agent's last message, as the provider recorded them. Idle time + /// between turns does not count. `None` when no turn times were recorded. + #[serde(default)] + pub active_seconds: Option, + /// Closed usage object. Cache-lifetime splits stay local; the ingest allowlist + /// does not include them. + #[serde(with = "session_usage_serde")] + #[schemars(with = "SessionUsageWire")] + pub usage: UsageCounts, + pub requests: u64, + /// Closed cost object. Currency, pricing source, and confidence stay local. + #[serde(with = "session_cost_serde")] + #[schemars(with = "SessionCostWire")] + pub cost: CostInfo, + #[serde(with = "session_models_serde")] + #[schemars(with = "Vec")] + pub models: Vec, + pub primary_model: Option, + pub total_messages: Option, + pub user_messages: Option, + pub assistant_messages: Option, + pub developer_messages: Option, + pub project: Option, + pub title: Option, + pub title_source: Option, + pub updated_at: DateTime, +} + +/// Usage keys accepted on `session_rollup.v1`. Five-minute and one-hour cache +/// splits are folded into `cache_creation_tokens` before this leaves the device. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema, Default)] +pub struct SessionUsageWire { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub local_prompt_eval_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub local_eval_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub requests: Option, +} + +/// Cost keys accepted on `session_rollup.v1`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema, Default)] +pub struct SessionCostWire { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_reported_usd: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub estimated_api_equivalent_usd: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_reported_micro_usd: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub estimated_api_equivalent_micro_usd: Option, +} + +/// One model row on `session_rollup.v1`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct SessionModelWire { + pub model: ModelInfo, + pub usage: SessionUsageWire, + pub cost: SessionCostWire, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metrics: Option, +} + +impl From<&UsageCounts> for SessionUsageWire { + fn from(usage: &UsageCounts) -> Self { + Self { + total_tokens: usage.total_tokens, + input_tokens: usage.input_tokens, + output_tokens: usage.output_tokens, + cache_creation_tokens: usage.cache_creation_tokens, + cache_read_tokens: usage.cache_read_tokens, + reasoning_tokens: usage.reasoning_tokens, + local_prompt_eval_tokens: usage.local_prompt_eval_tokens, + local_eval_tokens: usage.local_eval_tokens, + requests: usage.requests, + } + } +} + +impl From for UsageCounts { + fn from(usage: SessionUsageWire) -> Self { + Self { + input_tokens: usage.input_tokens, + output_tokens: usage.output_tokens, + cache_creation_tokens: usage.cache_creation_tokens, + cache_creation_5m_tokens: None, + cache_creation_1h_tokens: None, + cache_read_tokens: usage.cache_read_tokens, + reasoning_tokens: usage.reasoning_tokens, + total_tokens: usage.total_tokens, + requests: usage.requests, + local_prompt_eval_tokens: usage.local_prompt_eval_tokens, + local_eval_tokens: usage.local_eval_tokens, + } + } +} + +impl From<&CostInfo> for SessionCostWire { + fn from(cost: &CostInfo) -> Self { + Self { + provider_reported_usd: cost.provider_reported_usd, + estimated_api_equivalent_usd: cost.estimated_api_equivalent_usd, + provider_reported_micro_usd: cost.provider_reported_micro_usd, + estimated_api_equivalent_micro_usd: cost.estimated_api_equivalent_micro_usd, + } + } +} + +impl From for CostInfo { + fn from(cost: SessionCostWire) -> Self { + Self { + currency: "USD".to_string(), + estimated_api_equivalent_usd: cost.estimated_api_equivalent_usd, + provider_reported_usd: cost.provider_reported_usd, + estimated_api_equivalent_micro_usd: cost.estimated_api_equivalent_micro_usd, + provider_reported_micro_usd: cost.provider_reported_micro_usd, + pricing_source: None, + pricing_version: None, + confidence: Confidence::Medium, + } + } +} + +impl From<&SummaryModelUsage> for SessionModelWire { + fn from(usage: &SummaryModelUsage) -> Self { + Self { + model: usage.model.clone(), + usage: SessionUsageWire::from(&usage.usage), + cost: SessionCostWire::from(&usage.cost), + metrics: usage.metrics.clone(), + } + } +} + +impl From for SummaryModelUsage { + fn from(usage: SessionModelWire) -> Self { + Self { + model: usage.model, + usage: UsageCounts::from(usage.usage), + cost: CostInfo::from(usage.cost), + metrics: usage.metrics, + } + } +} + +mod session_usage_serde { + use super::{SessionUsageWire, UsageCounts}; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + pub(super) fn serialize(usage: &UsageCounts, serializer: S) -> Result + where + S: Serializer, + { + SessionUsageWire::from(usage).serialize(serializer) + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + Ok(UsageCounts::from(SessionUsageWire::deserialize( + deserializer, + )?)) + } +} + +mod session_cost_serde { + use super::{CostInfo, SessionCostWire}; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + pub(super) fn serialize(cost: &CostInfo, serializer: S) -> Result + where + S: Serializer, + { + SessionCostWire::from(cost).serialize(serializer) + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + Ok(CostInfo::from(SessionCostWire::deserialize(deserializer)?)) + } +} + +mod session_models_serde { + use super::{SessionModelWire, SummaryModelUsage}; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + pub(super) fn serialize( + models: &[SummaryModelUsage], + serializer: S, + ) -> Result + where + S: Serializer, + { + let wire = models + .iter() + .map(SessionModelWire::from) + .collect::>(); + wire.serialize(serializer) + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let wire = Vec::::deserialize(deserializer)?; + Ok(wire.into_iter().map(SummaryModelUsage::from).collect()) + } +} diff --git a/crates/statsai-core/src/types/sync.rs b/crates/statsai-core/src/types/sync.rs index 65edf24..d5bd18a 100644 --- a/crates/statsai-core/src/types/sync.rs +++ b/crates/statsai-core/src/types/sync.rs @@ -1,12 +1,12 @@ use crate::{ project_contains_file_paths, project_has_stable_identity, AccountEvidenceSummaryV1, AccountPlanProjectionV1, ActivityCoverageV1, ActivityRollupV1, CodeChangeMetric, ProjectInfo, - ProviderAccount, ProviderAccountId, QuotaCycleContributionV1, SourceAccountAssignment, - SourceAccountAssignmentId, SourceId, SourceLocation, Subscription, SubscriptionId, SummaryId, - TaskSpan, TaskVerification, TaskVerificationId, UsageEvent, UsageSummary, WorkItem, - WorkItemMember, + ProviderAccount, ProviderAccountId, QuotaCycleContributionV1, SessionRollupV1, + SourceAccountAssignment, SourceAccountAssignmentId, SourceId, SourceLocation, Subscription, + SubscriptionId, SummaryId, TaskSpan, TaskVerification, TaskVerificationId, UsageEvent, + UsageSummary, WorkItem, WorkItemMember, }; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, Datelike, Duration, Utc}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -56,6 +56,10 @@ pub struct SyncBatch { /// Per-source coverage for activity kinds, including zero-call days. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub activity_coverage: Vec, + /// Opt-in session rollups. Empty batches omit the key. Each row is + /// `session_rollup.v1` with the closed ingest key set. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub sessions: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub authoritative_snapshot: Option, pub created_at: DateTime, @@ -88,6 +92,11 @@ pub struct SyncAuthoritativeSnapshot { pub activity_rollup_ids: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub activity_coverage_ids: Vec, + /// Present only when this collector is opting into session retirement. + /// An empty array prunes every hosted session for the device. Absence + /// leaves hosted sessions alone, including on pre-session v6 collectors. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub session_rollup_ids: Option>, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -145,6 +154,9 @@ pub struct SyncEntityCounts { pub activity_rollups: u64, #[serde(default, skip_serializing_if = "sync_count_is_zero")] pub activity_coverage: u64, + /// Present on `sync_ack.v6`. Earlier acknowledgements omit it. + #[serde(default, skip_serializing_if = "sync_count_is_zero")] + pub sessions: u64, } fn sync_count_is_zero(value: &u64) -> bool { @@ -209,3 +221,301 @@ pub fn sanitize_summary_for_sync(mut summary: UsageSummary) -> UsageSummary { } summary } + +/// Ingest caps from `session_rollup.v1`. A value outside these bounds rejects +/// the whole batch, so the snapshot never lands and hosted rows stay put. +const SYNC_TOKEN_MAX: u64 = 1_000_000_000_000; +const SYNC_REQUEST_MAX: u64 = 1_000_000_000; +const SYNC_COST_MINOR_MAX: i64 = 1_000_000_000; +const SYNC_COST_MICRO_MAX: i64 = SYNC_COST_MINOR_MAX * 10_000; +const SYNC_SAFE_INT_MAX: u64 = (1_u64 << 53) - 1; +const SYNC_LABEL_UTF16_MAX: usize = 256; +const SYNC_PATH_LABEL_UTF16_MAX: usize = 1024; +const SYNC_SESSION_MODEL_MAX: usize = 64; +const SYNC_SESSION_PAYLOAD_MAX_BYTES: usize = 64 * 1024; +const SYNC_FUTURE_SKEW: Duration = Duration::seconds(24 * 60 * 60); + +/// Drops unstable projects and generic or secret-looking titles before a +/// session rollup leaves the device. `source_id` stays, matching summaries. +/// Nested usage and cost already serialize to the closed ingest keys. +/// +/// Values the ingest would reject are repaired here. One bad row fails the +/// batch, and session chunks are sent before the snapshot, so a permanent +/// rejection also leaves hosted rows that should have been retired. +#[must_use] +pub fn sanitize_session_rollup_for_sync(mut rollup: SessionRollupV1) -> SessionRollupV1 { + clamp_session_timestamps(&mut rollup); + rollup.project = rollup + .project + .and_then(sanitize_project_for_sync) + .map(clamp_project_labels); + clamp_session_title(&mut rollup); + rollup.primary_model = rollup + .primary_model + .and_then(|label| sync_label(&label, SYNC_LABEL_UTF16_MAX)); + clamp_usage_counts(&mut rollup.usage); + rollup.requests = rollup.requests.min(SYNC_REQUEST_MAX); + clamp_cost(&mut rollup.cost); + clamp_message_count(&mut rollup.total_messages); + clamp_message_count(&mut rollup.user_messages); + clamp_message_count(&mut rollup.assistant_messages); + clamp_message_count(&mut rollup.developer_messages); + for model in &mut rollup.models { + clamp_usage_counts(&mut model.usage); + clamp_cost(&mut model.cost); + model.model.name = model + .model + .name + .as_deref() + .and_then(|label| sync_label(label, SYNC_LABEL_UTF16_MAX)); + model.model.normalized_name = model + .model + .normalized_name + .as_deref() + .and_then(|label| sync_label(label, SYNC_LABEL_UTF16_MAX)); + model.model.provider_model_id = model + .model + .provider_model_id + .as_deref() + .and_then(|label| sync_label(label, SYNC_LABEL_UTF16_MAX)); + model.model.speed = model + .model + .speed + .as_deref() + .and_then(|label| sync_label(label, SYNC_LABEL_UTF16_MAX)); + model.model.reasoning_level_raw = model + .model + .reasoning_level_raw + .as_deref() + .and_then(|label| sync_label(label, SYNC_LABEL_UTF16_MAX)); + } + if rollup.models.len() > SYNC_SESSION_MODEL_MAX { + sort_models_by_tokens(&mut rollup.models); + rollup.models.truncate(SYNC_SESSION_MODEL_MAX); + } + shrink_session_payload(&mut rollup); + rollup +} + +fn clamp_session_timestamps(rollup: &mut SessionRollupV1) { + let latest = Utc::now() + SYNC_FUTURE_SKEW; + // Prefer a timestamp already on the row so a clock step does not give the + // repaired payload a new hash on every sync. + let fallback = [rollup.updated_at, rollup.started_at, rollup.ended_at] + .into_iter() + .find(|value| sync_timestamp_ok(*value, latest)) + .unwrap_or_else(|| Utc::now().min(latest)); + rollup.started_at = clamp_sync_timestamp(rollup.started_at, latest, fallback); + rollup.ended_at = clamp_sync_timestamp(rollup.ended_at, latest, fallback); + if rollup.ended_at < rollup.started_at { + rollup.ended_at = rollup.started_at; + } + rollup.updated_at = clamp_sync_timestamp(rollup.updated_at, latest, fallback); + rollup.active_seconds = rollup + .active_seconds + .map(|value| value.min(SYNC_SAFE_INT_MAX)); + // An unknown duration stays unknown; clamped timestamps must not turn it + // into a measured zero. + if rollup.duration_seconds.is_none() { + return; + } + let seconds = rollup + .ended_at + .signed_duration_since(rollup.started_at) + .num_seconds(); + rollup.duration_seconds = u64::try_from(seconds) + .ok() + .map(|value| value.min(SYNC_SAFE_INT_MAX)); + // Active time is a part of the span, so repaired timestamps bound it too. + if let (Some(active), Some(span)) = (rollup.active_seconds, rollup.duration_seconds) { + rollup.active_seconds = Some(active.min(span)); + } +} + +fn sync_timestamp_ok(value: DateTime, latest: DateTime) -> bool { + (1..=9999).contains(&value.year()) && value <= latest +} + +fn clamp_sync_timestamp( + value: DateTime, + latest: DateTime, + fallback: DateTime, +) -> DateTime { + if sync_timestamp_ok(value, latest) { + value + } else { + fallback + } +} + +fn clamp_project_labels(mut project: ProjectInfo) -> ProjectInfo { + project.project_label = project + .project_label + .as_deref() + .and_then(|label| sync_label(label, SYNC_LABEL_UTF16_MAX)); + project.repo_label = project + .repo_label + .as_deref() + .and_then(|label| sync_label(label, SYNC_LABEL_UTF16_MAX)); + project.branch_label = project + .branch_label + .as_deref() + .and_then(|label| sync_label(label, SYNC_LABEL_UTF16_MAX)); + project.path_label = project + .path_label + .as_deref() + .and_then(|label| sync_label(label, SYNC_PATH_LABEL_UTF16_MAX)); + project +} + +fn clamp_session_title(rollup: &mut SessionRollupV1) { + let title = rollup + .title + .as_deref() + .and_then(|label| sync_label(label, SYNC_LABEL_UTF16_MAX)) + .filter(|label| match rollup.title_source { + // The provider named the session; keep its name unless unsafe. + Some(crate::SessionTitleSource::Event) => { + !crate::provider_session_title_is_unusable(Some(label)) + } + _ => !crate::task_title_is_generic(Some(label)), + }); + if title.is_none() { + rollup.title_source = None; + } + rollup.title = title; +} + +fn clamp_usage_counts(usage: &mut crate::UsageCounts) { + clamp_token(&mut usage.input_tokens); + clamp_token(&mut usage.output_tokens); + clamp_token(&mut usage.cache_creation_tokens); + clamp_token(&mut usage.cache_creation_5m_tokens); + clamp_token(&mut usage.cache_creation_1h_tokens); + clamp_token(&mut usage.cache_read_tokens); + clamp_token(&mut usage.reasoning_tokens); + clamp_token(&mut usage.total_tokens); + clamp_token(&mut usage.local_prompt_eval_tokens); + clamp_token(&mut usage.local_eval_tokens); + if let Some(requests) = usage.requests.as_mut() { + *requests = (*requests).min(SYNC_REQUEST_MAX); + } +} + +fn clamp_token(value: &mut Option) { + if let Some(amount) = value.as_mut() { + *amount = (*amount).min(SYNC_TOKEN_MAX); + } +} + +fn clamp_message_count(value: &mut Option) { + if let Some(amount) = value.as_mut() { + *amount = (*amount).min(SYNC_SAFE_INT_MAX); + } +} + +fn clamp_cost(cost: &mut crate::CostInfo) { + clamp_cost_field(&mut cost.provider_reported_usd, SYNC_COST_MINOR_MAX); + clamp_cost_field(&mut cost.estimated_api_equivalent_usd, SYNC_COST_MINOR_MAX); + clamp_cost_field(&mut cost.provider_reported_micro_usd, SYNC_COST_MICRO_MAX); + clamp_cost_field( + &mut cost.estimated_api_equivalent_micro_usd, + SYNC_COST_MICRO_MAX, + ); +} + +fn clamp_cost_field(value: &mut Option, max: i64) { + if let Some(amount) = value.as_mut() { + *amount = (*amount).clamp(0, max); + } +} + +fn sort_models_by_tokens(models: &mut [crate::SummaryModelUsage]) { + models.sort_by(|left, right| { + right + .usage + .computed_total() + .cmp(&left.usage.computed_total()) + .then_with(|| model_wire_key(&left.model).cmp(&model_wire_key(&right.model))) + }); +} + +fn model_wire_key(model: &crate::ModelInfo) -> String { + model + .normalized_name + .clone() + .or_else(|| model.name.clone()) + .or_else(|| model.provider_model_id.clone()) + .unwrap_or_default() +} + +fn shrink_session_payload(rollup: &mut SessionRollupV1) { + if payload_within_limit(rollup) { + return; + } + if rollup.models.len() > 1 { + sort_models_by_tokens(&mut rollup.models); + } + while rollup.models.len() > 1 && !payload_within_limit(rollup) { + rollup.models.pop(); + } + if payload_within_limit(rollup) { + return; + } + if let Some(project) = rollup.project.as_mut() { + project.path_label = None; + project.project_label = None; + project.repo_label = None; + project.branch_label = None; + } + rollup.title = None; + rollup.title_source = None; + if !payload_within_limit(rollup) { + rollup.models.clear(); + } +} + +fn payload_within_limit(rollup: &SessionRollupV1) -> bool { + serde_json::to_vec(rollup) + .map(|payload| payload.len() <= SYNC_SESSION_PAYLOAD_MAX_BYTES) + .unwrap_or(false) +} + +fn sync_label(value: &str, max_utf16: usize) -> Option { + let cleaned = value + .chars() + .map(|ch| { + if is_disallowed_sync_control(ch) { + ' ' + } else { + ch + } + }) + .collect::(); + let collapsed = cleaned.split_whitespace().collect::>().join(" "); + let truncated = truncate_utf16(collapsed.trim(), max_utf16); + let truncated = truncated.trim(); + if truncated.is_empty() { + None + } else { + Some(truncated.to_string()) + } +} + +fn is_disallowed_sync_control(ch: char) -> bool { + ('\u{0000}'..='\u{001f}').contains(&ch) || ch == '\u{007f}' +} + +fn truncate_utf16(value: &str, max_units: usize) -> String { + let mut units = 0usize; + let mut end = 0usize; + for (index, ch) in value.char_indices() { + let next = units + ch.len_utf16(); + if next > max_units { + break; + } + units = next; + end = index + ch.len_utf8(); + } + value[..end].to_string() +} diff --git a/crates/statsai-daemon/src/ingest.rs b/crates/statsai-daemon/src/ingest.rs index 3b3211b..0153668 100644 --- a/crates/statsai-daemon/src/ingest.rs +++ b/crates/statsai-daemon/src/ingest.rs @@ -65,6 +65,9 @@ pub fn ingest_sync_batch(store: &Store, batch: &SyncBatch) -> Result { { bail!("activity collections require sync_batch.v6"); } + if batch.schema_version != SYNC_BATCH_V6_SCHEMA_VERSION && !batch.sessions.is_empty() { + bail!("session rollups require sync_batch.v6"); + } if batch .code_change_metrics .iter() @@ -91,6 +94,9 @@ pub fn ingest_sync_batch(store: &Store, batch: &SyncBatch) -> Result { if !batch.activity_rollups.is_empty() || !batch.activity_coverage.is_empty() { bail!("activity collections are not supported by the loopback daemon"); } + if !batch.sessions.is_empty() { + bail!("session rollups are not supported by the loopback daemon"); + } let result = store.ingest_sync_batch(batch)?; @@ -112,6 +118,7 @@ pub fn ingest_sync_batch(store: &Store, batch: &SyncBatch) -> Result { account_evidence_summaries: batch.account_evidence_summaries.len() as u64, activity_rollups: 0, activity_coverage: 0, + sessions: 0, }, duplicates: SyncEntityCounts { sources: 0, @@ -129,6 +136,7 @@ pub fn ingest_sync_batch(store: &Store, batch: &SyncBatch) -> Result { account_evidence_summaries: 0, activity_rollups: 0, activity_coverage: 0, + sessions: 0, }, rejected: Vec::::new(), }) diff --git a/crates/statsai-daemon/src/tests.rs b/crates/statsai-daemon/src/tests.rs index 8df1c6d..040055a 100644 --- a/crates/statsai-daemon/src/tests.rs +++ b/crates/statsai-daemon/src/tests.rs @@ -24,6 +24,7 @@ fn empty_batch() -> SyncBatch { account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: Vec::new(), @@ -329,6 +330,62 @@ fn ingest_v6_batch_refuses_activity_collections_it_cannot_store() { .contains("activity collections are not supported")); } +#[test] +fn ingest_refuses_session_rollups_the_loopback_daemon_cannot_store() { + let store = Store::in_memory().expect("store"); + let mut batch = empty_batch(); + batch.schema_version = SYNC_BATCH_V5_SCHEMA_VERSION.to_string(); + batch.sessions.push(loopback_session(&batch.device_id)); + let error = ingest_sync_batch(&store, &batch).expect_err("pre-v6 sessions"); + assert!(error + .to_string() + .contains("session rollups require sync_batch.v6")); + + batch.schema_version = SYNC_BATCH_V6_SCHEMA_VERSION.to_string(); + let error = ingest_sync_batch(&store, &batch).expect_err("unsupported sessions"); + assert!(error + .to_string() + .contains("session rollups are not supported")); +} + +fn loopback_session(device_id: &str) -> statsai_core::SessionRollupV1 { + let now = Utc::now(); + statsai_core::SessionRollupV1 { + schema_version: statsai_core::SESSION_ROLLUP_SCHEMA_VERSION.to_string(), + session_id: "session_loopback".to_string(), + device_id: device_id.to_string(), + provider: "codex".to_string(), + source_id: statsai_core::SourceId("source".to_string()), + provider_account_id: None, + started_at: now, + ended_at: now, + duration_seconds: Some(1), + active_seconds: None, + usage: UsageCounts::default(), + requests: 1, + cost: statsai_core::CostInfo { + currency: "USD".to_string(), + estimated_api_equivalent_usd: None, + provider_reported_usd: None, + estimated_api_equivalent_micro_usd: None, + provider_reported_micro_usd: None, + pricing_source: None, + pricing_version: None, + confidence: Confidence::Medium, + }, + models: Vec::new(), + primary_model: None, + total_messages: None, + user_messages: None, + assistant_messages: None, + developer_messages: None, + project: None, + title: None, + title_source: None, + updated_at: now, + } +} + #[test] fn ingest_v3_batch_rejects_metric_owned_by_another_device_before_persisting() { let store = Store::in_memory().expect("store"); diff --git a/crates/statsai-daemon/src/watch/scan.rs b/crates/statsai-daemon/src/watch/scan.rs index d144a74..d0aa4b4 100644 --- a/crates/statsai-daemon/src/watch/scan.rs +++ b/crates/statsai-daemon/src/watch/scan.rs @@ -491,6 +491,7 @@ fn rescan_changed_sources_with_adapters_and_commit_store_and_dependencies( reconciled_file_hashes: &reconciled_file_hashes, events: &scan.events, summaries: &scan.summaries, + session_names: &scan.session_names, activity_invocations: &scan.activity_invocations, activity_coverage: &scan.activity_coverage, activity_persist_mode: ActivityPersistMode::for_scan( diff --git a/crates/statsai-store/src/account_plan/observations.rs b/crates/statsai-store/src/account_plan/observations.rs index f177570..10dd3ea 100644 --- a/crates/statsai-store/src/account_plan/observations.rs +++ b/crates/statsai-store/src/account_plan/observations.rs @@ -571,15 +571,16 @@ impl Store { .collect::>(); self.apply_conversation_account_bindings(source_id, &mut events)?; let mut changed = 0u64; - let mut dirty_keys = BTreeSet::new(); + let mut dirty = crate::events::EventWriteDirty::default(); for (event, previous_account) in events.iter().zip(previous_accounts) { if event.provider_account_id == previous_account { continue; } - dirty_keys.extend(self.update_event_payload(event)?); + dirty.extend(self.update_event_payload(event)?); changed += 1; } - self.refresh_sync_rollups_for_keys(&dirty_keys)?; + self.refresh_sync_rollups_for_keys(&dirty.buckets)?; + self.refresh_session_rollups_for_keys(&dirty.sessions)?; Ok(changed) } diff --git a/crates/statsai-store/src/accounts.rs b/crates/statsai-store/src/accounts.rs index 8a19e82..1071dbf 100644 --- a/crates/statsai-store/src/accounts.rs +++ b/crates/statsai-store/src/accounts.rs @@ -360,10 +360,17 @@ impl Store { .map(|value| parse_bool_metadata_value(SYNC_INCLUDE_ACTIVITY_METADATA_KEY, value)) .transpose()? .unwrap_or(false); + let include_sessions = self + .metadata_value(SYNC_INCLUDE_SESSIONS_METADATA_KEY)? + .as_deref() + .map(|value| parse_bool_metadata_value(SYNC_INCLUDE_SESSIONS_METADATA_KEY, value)) + .transpose()? + .unwrap_or(false); Ok(SyncPreferences { include_projects, include_tasks, include_activity, + include_sessions, } .normalized()) } @@ -390,6 +397,14 @@ impl Store { "0" }, )?; + self.set_metadata_value( + SYNC_INCLUDE_SESSIONS_METADATA_KEY, + if preferences.include_sessions { + "1" + } else { + "0" + }, + )?; Ok(()) } } diff --git a/crates/statsai-store/src/archive/write.rs b/crates/statsai-store/src/archive/write.rs index 873d591..99543e0 100644 --- a/crates/statsai-store/src/archive/write.rs +++ b/crates/statsai-store/src/archive/write.rs @@ -34,8 +34,8 @@ impl Store { INSERT INTO archive_conversations (conversation_id, provider, source_id, native_conversation_id, title, project_json, started_at, updated_at, completeness, - missing_content_count, imported_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) + missing_content_count, imported_at, session_hash) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12) ON CONFLICT(conversation_id) DO UPDATE SET source_id = excluded.source_id, title = COALESCE(excluded.title, archive_conversations.title), @@ -68,6 +68,7 @@ impl Store { conversation.completeness.as_str(), conversation.missing_content_count, &imported_at, + statsai_core::hash_text(&conversation.native_conversation_id), ], )?; result.conversations += 1; @@ -201,6 +202,11 @@ impl Store { ], )?; } + let raw_session_ids = conversations + .iter() + .map(|conversation| conversation.native_conversation_id.clone()) + .collect::>(); + self.refresh_session_rollups_for_raw_ids(&raw_session_ids)?; Ok(result) } diff --git a/crates/statsai-store/src/code_changes/tests/refresh.rs b/crates/statsai-store/src/code_changes/tests/refresh.rs index e3dddea..16af4fa 100644 --- a/crates/statsai-store/src/code_changes/tests/refresh.rs +++ b/crates/statsai-store/src/code_changes/tests/refresh.rs @@ -434,6 +434,7 @@ fn local_refresh_preserves_metrics_ingested_from_another_device() { account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: Vec::new(), diff --git a/crates/statsai-store/src/events.rs b/crates/statsai-store/src/events.rs index 77680d7..92483f8 100644 --- a/crates/statsai-store/src/events.rs +++ b/crates/statsai-store/src/events.rs @@ -1,11 +1,35 @@ use super::*; use crate::dedupe::*; +#[derive(Debug, Default)] +pub(crate) struct EventWriteDirty { + pub(crate) buckets: BTreeSet, + pub(crate) sessions: BTreeSet, +} + +impl EventWriteDirty { + fn from_event(event: &UsageEvent) -> Self { + let mut dirty = Self::default(); + dirty.observe(event); + dirty + } + + fn observe(&mut self, event: &UsageEvent) { + self.buckets.insert(sync_rollup_bucket_key(event)); + self.sessions.insert(event.session.session_id.clone()); + } + + pub(crate) fn extend(&mut self, other: Self) { + self.buckets.extend(other.buckets); + self.sessions.extend(other.sessions); + } +} + #[derive(Debug)] struct EventInsertOutcome { inserted: bool, canonical_event_id: EventId, - dirty_keys: BTreeSet, + dirty: EventWriteDirty, } impl Store { @@ -16,8 +40,8 @@ impl Store { let existing = self.event_by_id(&existing_id)?; let refreshed = refreshed_duplicate_event(existing.as_ref(), &event, existing_id.as_str()); - let dirty_keys = self.update_event_payload(&refreshed)?; - self.refresh_sync_rollups_for_keys(&dirty_keys)?; + let dirty = self.update_event_payload(&refreshed)?; + self.refresh_event_rollups(&dirty)?; return Ok(false); } @@ -45,10 +69,10 @@ impl Store { let existing = self.event_by_id(&event.event_id.0)?; let refreshed = refreshed_duplicate_event(existing.as_ref(), &event, event.event_id.0.as_str()); - let dirty_keys = self.update_event_payload(&refreshed)?; - self.refresh_sync_rollups_for_keys(&dirty_keys)?; + let dirty = self.update_event_payload(&refreshed)?; + self.refresh_event_rollups(&dirty)?; } else { - self.refresh_sync_rollups_for_keys(&BTreeSet::from([sync_rollup_bucket_key(&event)]))?; + self.refresh_event_rollups(&EventWriteDirty::from_event(&event))?; } Ok(changed > 0) } @@ -75,7 +99,7 @@ impl Store { let mut conflict_map = self.batch_load_conflicts(&conflict_keys)?; let mut inserted = 0u64; let mut canonical_event_ids = HashMap::with_capacity(events.len()); - let mut dirty_keys = BTreeSet::new(); + let mut dirty = EventWriteDirty::default(); for (index, event) in events.iter().enumerate() { let incoming_event_id = event.event_id.clone(); let matched_event = @@ -99,7 +123,7 @@ impl Store { if let Some((existing_id, existing)) = matched_event { let refreshed = refreshed_duplicate_event(Some(&existing), event, existing_id.as_str()); - dirty_keys.extend(self.update_event_payload(&refreshed)?); + dirty.extend(self.update_event_payload(&refreshed)?); canonical_event_ids.insert(incoming_event_id, EventId(existing_id.clone())); let candidates = conflict_map .entry(conflict_keys[index].clone()) @@ -123,7 +147,7 @@ impl Store { inserted += 1; } canonical_event_ids.insert(incoming_event_id, outcome.canonical_event_id.clone()); - dirty_keys.extend(outcome.dirty_keys); + dirty.extend(outcome.dirty); conflict_map .entry(conflict_keys[index].clone()) .or_default() @@ -132,7 +156,7 @@ impl Store { event: event.clone(), }); } - self.refresh_sync_rollups_for_keys(&dirty_keys)?; + self.refresh_event_rollups(&dirty)?; Ok(EventInsertBatchResult { inserted, canonical_event_ids, @@ -140,20 +164,20 @@ impl Store { }) } - pub(crate) fn update_event_payload( - &self, - event: &UsageEvent, - ) -> Result> { - let existing_bucket = self - .event_by_id(&event.event_id.0)? - .map(|existing| sync_rollup_bucket_key(&existing)); - let bucket = self.update_event_cost_payload(event)?; - let mut dirty_keys = BTreeSet::new(); - if let Some(existing_bucket) = existing_bucket { - dirty_keys.insert(existing_bucket); + fn refresh_event_rollups(&self, dirty: &EventWriteDirty) -> Result<()> { + self.refresh_sync_rollups_for_keys(&dirty.buckets)?; + self.refresh_session_rollups_for_keys(&dirty.sessions)?; + Ok(()) + } + + pub(crate) fn update_event_payload(&self, event: &UsageEvent) -> Result { + let existing = self.event_by_id(&event.event_id.0)?; + self.update_event_cost_payload(event)?; + let mut dirty = EventWriteDirty::from_event(event); + if let Some(existing) = existing.as_ref() { + dirty.observe(existing); } - dirty_keys.insert(bucket); - Ok(dirty_keys) + Ok(dirty) } /// Updates a persisted event's payload without re-reading it. @@ -224,13 +248,13 @@ impl Store { return Ok(EventInsertOutcome { inserted: false, canonical_event_id: event.event_id.clone(), - dirty_keys: self.update_event_payload(&refreshed)?, + dirty: self.update_event_payload(&refreshed)?, }); } Ok(EventInsertOutcome { inserted: true, canonical_event_id: event.event_id.clone(), - dirty_keys: BTreeSet::from([sync_rollup_bucket_key(event)]), + dirty: EventWriteDirty::from_event(event), }) } @@ -569,12 +593,12 @@ impl Store { } self.with_immediate_transaction(|| { let mut changed = 0u64; - let mut dirty_keys = BTreeSet::new(); + let mut dirty = EventWriteDirty::default(); for event in events { - dirty_keys.extend(self.update_event_payload(event)?); + dirty.extend(self.update_event_payload(event)?); changed += 1; } - self.refresh_sync_rollups_for_keys(&dirty_keys)?; + self.refresh_event_rollups(&dirty)?; Ok(changed) }) } @@ -585,6 +609,20 @@ impl Store { ) -> Result { self.with_immediate_transaction(|| { let mut impact = EventDeletionImpact::default(); + // A session can hold events from more than one source, so the + // sessions these events belong to are rebuilt from what remains + // rather than deleted by the source their newest event came from. + let mut session_ids = BTreeSet::new(); + for source_id in source_ids { + let mut statement = self.conn.prepare(&format!( + "SELECT DISTINCT {EVENT_SESSION_ID_SQL} FROM usage_events WHERE source_id = ?1" + ))?; + let rows = statement + .query_map(params![&source_id.0], |row| row.get::<_, Option>(0))?; + for row in rows { + session_ids.extend(row?); + } + } for source_id in source_ids { // `usage_events_source_idx` covers this, so naming the ids costs an // index range read rather than a pass over the payloads. @@ -605,6 +643,7 @@ impl Store { )? as u64; } self.delete_sync_rollups_for_sources_in_tx(source_ids)?; + self.refresh_session_rollups_for_keys(&session_ids)?; Ok(impact) }) } @@ -620,7 +659,7 @@ impl Store { self.with_immediate_transaction(|| { let mut impact = EventDeletionImpact::default(); - let mut dirty_keys = BTreeSet::new(); + let mut dirty = EventWriteDirty::default(); // Both statements filter on the expression `usage_events_source_file_idx` // indexes, so they read an index range rather than the whole table. @@ -643,7 +682,7 @@ impl Store { for payload in payloads { let event: UsageEvent = serde_json::from_str(&payload)?; - dirty_keys.insert(sync_rollup_bucket_key(&event)); + dirty.observe(&event); impact.deleted_event_ids.push(event.event_id); } @@ -653,7 +692,7 @@ impl Store { as u64; } - self.refresh_sync_rollups_for_keys(&dirty_keys)?; + self.refresh_event_rollups(&dirty)?; Ok(impact) }) } diff --git a/crates/statsai-store/src/lib.rs b/crates/statsai-store/src/lib.rs index f34c59b..3bc5eb7 100644 --- a/crates/statsai-store/src/lib.rs +++ b/crates/statsai-store/src/lib.rs @@ -26,7 +26,7 @@ pub(crate) use rollups::{ collect_pending_summary_days, event_with_valid_project, is_daily_rollup_summary, is_http_rollup_passthrough_summary, sanitize_summary_for_http_sync, summary_period_bounds, summary_sync_payload_hash, sync_rollup_bucket_key, sync_rollup_project_key, - SyncRollupBucketKey, + task_span_session_hashes, SyncRollupBucketKey, }; pub(crate) use sql::{ begin_immediate_transaction_with_retry, commit_transaction, enable_sqlite_defensive, @@ -80,6 +80,7 @@ pub use pricing::{ apply_current_estimated_pricing, RepricingReport, APPLIED_PRICING_CATALOG_VERSION_KEY, APPLIED_PRICING_RULESET_VERSION_KEY, }; +pub use rollups::{SessionFilter, SessionSort, SessionStats}; pub use snapshot::{ clone_database_to, database_applied_pricing_ruleset_version, database_schema_version, DatabaseClone, @@ -113,6 +114,7 @@ const SYNC_ROLLUP_SUMMARY_VERSION: &str = "14"; const SYNC_INCLUDE_PROJECTS_METADATA_KEY: &str = "sync.include_projects"; const SYNC_INCLUDE_TASKS_METADATA_KEY: &str = "sync.include_tasks"; const SYNC_INCLUDE_ACTIVITY_METADATA_KEY: &str = "sync.include_activity"; +const SYNC_INCLUDE_SESSIONS_METADATA_KEY: &str = "sync.include_sessions"; const LEGACY_CODEX_PLAN_CONVERSION_METADATA_KEY: &str = "migration.legacy_codex_plan_evidence.v1"; const SQLITE_BUSY_TIMEOUT: Duration = if cfg!(test) { Duration::from_millis(50) @@ -142,6 +144,10 @@ pub(crate) const EVENT_SOURCE_FILE_HASH_SQL: &str = "CASE WHEN json_valid(payloa THEN json_extract(payload, '$.parse_evidence.source_file_path_hash') END"; pub(crate) const EVENT_CONVERSATION_HASH_SQL: &str = "CASE WHEN json_valid(payload) \ THEN json_extract(payload, '$.session.local_session_id_hash') END"; +/// Hashed `session_…` id. Indexed so a session refresh reads one conversation +/// instead of scanning every usage payload. +pub(crate) const EVENT_SESSION_ID_SQL: &str = "CASE WHEN json_valid(payload) \ + THEN json_extract(payload, '$.session.session_id') END"; /// The one field plan evidence needs out of a quota observation payload. pub(crate) const QUOTA_PLAN_TYPE_SQL: &str = "CASE WHEN json_valid(payload) \ THEN json_extract(payload, '$.status.plan_type') END"; @@ -188,17 +194,19 @@ pub struct SyncPreferences { pub include_projects: bool, pub include_tasks: bool, pub include_activity: bool, + pub include_sessions: bool, } impl SyncPreferences { #[must_use] pub fn normalized(self) -> Self { - let include_projects = self.include_projects || self.include_tasks; + let include_projects = self.include_projects || self.include_tasks || self.include_sessions; let include_tasks = self.include_tasks && include_projects; Self { include_projects, include_tasks, include_activity: self.include_activity, + include_sessions: self.include_sessions, } } } @@ -282,6 +290,7 @@ pub struct ScanFileReplacement<'a> { pub reconciled_file_hashes: &'a [String], pub events: &'a [UsageEvent], pub summaries: &'a [UsageSummary], + pub session_names: &'a [statsai_core::SessionName], pub activity_invocations: &'a [ActivityInvocationV1], pub activity_coverage: &'a [ActivityCoverageV1], pub activity_persist_mode: ActivityPersistMode, @@ -355,6 +364,7 @@ impl Store { let store = Self { conn }; store.migrate()?; store.configure_connection()?; + store.backfill_session_rollups_if_needed()?; store.conn.execute_batch("PRAGMA optimize=0x10002;")?; Ok(store) } @@ -436,6 +446,7 @@ impl Store { store.conn.busy_timeout(SQLITE_BUSY_TIMEOUT)?; store.migrate()?; store.configure_connection()?; + store.backfill_session_rollups_if_needed()?; store.conn.execute_batch("PRAGMA optimize=0x10002;")?; Ok(store) } diff --git a/crates/statsai-store/src/migrations/mod.rs b/crates/statsai-store/src/migrations/mod.rs index d4887d6..6d6b9a1 100644 --- a/crates/statsai-store/src/migrations/mod.rs +++ b/crates/statsai-store/src/migrations/mod.rs @@ -8,7 +8,7 @@ mod v2; pub(crate) use v1::*; pub(crate) use v2::*; -pub const CURRENT_SCHEMA_VERSION: i64 = 25; +pub const CURRENT_SCHEMA_VERSION: i64 = 28; pub fn migrate(conn: &Connection) -> Result<()> { if let Some(current) = existing_schema_version(conn)? { @@ -150,6 +150,9 @@ fn apply_migration(conn: &Connection, version: i64) -> Result<()> { 23 => apply_migration_023(conn), 24 => apply_migration_024(conn), 25 => apply_migration_025(conn), + 26 => apply_migration_026(conn), + 27 => apply_migration_027(conn), + 28 => apply_migration_028(conn), _ => bail!("unsupported schema migration version {version}"), } } @@ -390,6 +393,18 @@ mod tests { "collect a payload's remaining references", "SELECT 1 FROM quota_observations o WHERE o.payload_hash = ?1".to_string(), ), + ( + "look up a session's archive titles", + "SELECT title FROM archive_conversations WHERE session_hash IN (?1, ?2)" + .to_string(), + ), + ( + "look up a session's task-span titles", + "SELECT title FROM task_spans + WHERE is_meta = 0 + AND (session_hash IN (?1, ?2) OR session_uuid_hash IN (?1, ?2))" + .to_string(), + ), ] { let plan = query_plan(&conn, &sql); assert!( @@ -442,21 +457,27 @@ mod tests { version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL ); - INSERT INTO schema_migrations (version, applied_at) - VALUES (26, '2026-08-23T00:00:00Z'); "#, ) + .expect("prepare future schema marker"); + let future = CURRENT_SCHEMA_VERSION + 1; + conn.execute( + "INSERT INTO schema_migrations (version, applied_at) VALUES (?1, '2026-08-23T00:00:00Z')", + [future], + ) .expect("create future schema marker"); - let error = migrate(&conn).expect_err("schema 26 must be rejected by schema 25 binary"); + let error = migrate(&conn).expect_err("newer schema must be rejected"); assert_eq!( error.to_string(), - "database schema version 26 is newer than this StatsAI binary supports (25); upgrade StatsAI or use a compatible database" + format!( + "database schema version {future} is newer than this StatsAI binary supports ({CURRENT_SCHEMA_VERSION}); upgrade StatsAI or use a compatible database" + ) ); assert_eq!( current_schema_version(&conn).expect("read unchanged version"), - 26 + future ); } @@ -502,6 +523,68 @@ mod tests { )); } + #[test] + fn version_twenty_seven_backfills_session_title_keys() { + let conn = Connection::open_in_memory().expect("open in-memory database"); + ensure_migrations_table(&conn).expect("ensure migrations table"); + for version in 1..=27 { + apply_migration(&conn, version).expect("apply earlier migration"); + record_migration(&conn, version).expect("record earlier migration"); + } + let rollout = + "sessions/2026/06/01/rollout-2026-06-01T08-00-00-0199a0b1-c2d3-7e4f-8a9b-0c1d2e3f4a5b"; + conn.execute_batch(&format!( + r#" + INSERT INTO task_spans (span_id, provider, source_id, project_bucket, started_at, + title, normalized_title, confidence, payload) + VALUES + ('span-rollout', 'codex', 'source-1', 'bucket', '2026-06-01T08:00:00Z', + 'Fix parser', 'fix parser', 'high', '{{"session_id":"{rollout}"}}'), + ('span-none', 'codex', 'source-1', 'bucket', '2026-06-01T08:00:00Z', + 'Other', 'other', 'high', '{{}}'); + INSERT INTO archive_conversations (conversation_id, provider, source_id, + native_conversation_id, completeness, imported_at) + VALUES ('conversation-1', 'claude_code', 'source-2', 'claude-session', 'complete', + '2026-06-01T08:00:00Z'); + "# + )) + .expect("seed version twenty-seven rows"); + + migrate(&conn).expect("migrate version twenty-seven database"); + + let span_keys = |span_id: &str| { + conn.query_row( + "SELECT session_hash, session_uuid_hash FROM task_spans WHERE span_id = ?1", + [span_id], + |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, Option>(1)?, + )) + }, + ) + .expect("span keys") + }; + assert_eq!( + span_keys("span-rollout"), + ( + Some(statsai_core::hash_text(rollout)), + Some(statsai_core::hash_text( + "0199a0b1-c2d3-7e4f-8a9b-0c1d2e3f4a5b" + )), + ) + ); + assert_eq!(span_keys("span-none"), (None, None)); + let archive_key: String = conn + .query_row( + "SELECT session_hash FROM archive_conversations WHERE conversation_id = 'conversation-1'", + [], + |row| row.get(0), + ) + .expect("archive key"); + assert_eq!(archive_key, statsai_core::hash_text("claude-session")); + } + #[test] fn version_eleven_archive_receives_source_record_index() { let conn = Connection::open_in_memory().expect("open in-memory database"); diff --git a/crates/statsai-store/src/migrations/v2.rs b/crates/statsai-store/src/migrations/v2.rs index 8f87d33..1b64771 100644 --- a/crates/statsai-store/src/migrations/v2.rs +++ b/crates/statsai-store/src/migrations/v2.rs @@ -1,5 +1,8 @@ use super::*; -use crate::{EVENT_CONVERSATION_HASH_SQL, EVENT_SOURCE_FILE_HASH_SQL, QUOTA_PLAN_TYPE_SQL}; +use crate::{ + EVENT_CONVERSATION_HASH_SQL, EVENT_SESSION_ID_SQL, EVENT_SOURCE_FILE_HASH_SQL, + QUOTA_PLAN_TYPE_SQL, +}; use rusqlite::params; use statsai_core::{account_plan_observation_id, AccountPlanObservationV1}; @@ -548,3 +551,107 @@ pub(crate) fn apply_migration_025(conn: &Connection) -> Result<()> { )?; Ok(()) } + +pub(crate) fn apply_migration_026(conn: &Connection) -> Result<()> { + conn.execute_batch(&format!( + r#" + CREATE TABLE IF NOT EXISTS session_rollups ( + session_id TEXT PRIMARY KEY, + provider TEXT NOT NULL, + source_id TEXT NOT NULL, + provider_account_id TEXT, + day_key TEXT NOT NULL, + started_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + total_tokens INTEGER NOT NULL, + payload_hash TEXT NOT NULL, + dirty INTEGER NOT NULL DEFAULT 1, + payload TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS session_rollups_started_idx + ON session_rollups (started_at); + CREATE INDEX IF NOT EXISTS session_rollups_provider_day_idx + ON session_rollups (provider, day_key); + CREATE INDEX IF NOT EXISTS usage_events_session_idx + ON usage_events ({EVENT_SESSION_ID_SQL}); + "#, + ))?; + Ok(()) +} + +/// Provider-held session names, keyed by the hashed session id. Session +/// rollups read them ahead of titles stamped on events, so a rename reaches the +/// session without re-parsing its transcripts. +pub(crate) fn apply_migration_027(conn: &Connection) -> Result<()> { + conn.execute_batch( + r#" + CREATE TABLE IF NOT EXISTS session_names ( + local_session_id_hash TEXT PRIMARY KEY, + title TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + "#, + )?; + Ok(()) +} + +/// Hashed session keys on task spans and archive conversations, so a session +/// refresh looks up the titles of its own sessions instead of reading both +/// tables whole. Session names also record the parent a sub-agent borrows its +/// name from. +pub(crate) fn apply_migration_028(conn: &Connection) -> Result<()> { + conn.execute_batch( + r#" + ALTER TABLE task_spans ADD COLUMN session_hash TEXT; + ALTER TABLE task_spans ADD COLUMN session_uuid_hash TEXT; + ALTER TABLE archive_conversations ADD COLUMN session_hash TEXT; + ALTER TABLE session_names ADD COLUMN parent_hash TEXT; + CREATE INDEX IF NOT EXISTS session_names_parent_idx + ON session_names (parent_hash) WHERE parent_hash IS NOT NULL; + "#, + )?; + let spans = { + let mut statement = conn.prepare( + "SELECT span_id, json_extract(payload, '$.session_id') FROM task_spans + WHERE json_extract(payload, '$.session_id') IS NOT NULL", + )?; + let rows = statement.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })?; + rows.collect::>>()? + }; + let mut update_span = conn.prepare( + "UPDATE task_spans SET session_hash = ?2, session_uuid_hash = ?3 WHERE span_id = ?1", + )?; + for (span_id, session_id) in spans { + let (session_hash, session_uuid_hash) = crate::task_span_session_hashes(Some(&session_id)); + update_span.execute(params![span_id, session_hash, session_uuid_hash])?; + } + let conversations = { + let mut statement = conn + .prepare("SELECT conversation_id, native_conversation_id FROM archive_conversations")?; + let rows = statement.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })?; + rows.collect::>>()? + }; + let mut update_conversation = conn + .prepare("UPDATE archive_conversations SET session_hash = ?2 WHERE conversation_id = ?1")?; + for (conversation_id, native_id) in conversations { + update_conversation.execute(params![ + conversation_id, + statsai_core::hash_text(&native_id) + ])?; + } + conn.execute_batch( + r#" + CREATE INDEX IF NOT EXISTS task_spans_session_hash_idx + ON task_spans (session_hash) WHERE session_hash IS NOT NULL; + CREATE INDEX IF NOT EXISTS task_spans_session_uuid_hash_idx + ON task_spans (session_uuid_hash) WHERE session_uuid_hash IS NOT NULL; + CREATE INDEX IF NOT EXISTS archive_conversations_session_hash_idx + ON archive_conversations (session_hash); + "#, + )?; + Ok(()) +} diff --git a/crates/statsai-store/src/pricing/mod.rs b/crates/statsai-store/src/pricing/mod.rs index 01225b3..d0b0abe 100644 --- a/crates/statsai-store/src/pricing/mod.rs +++ b/crates/statsai-store/src/pricing/mod.rs @@ -130,10 +130,12 @@ impl Store { let mut report = RepricingReport::default(); let mut dirty_keys = BTreeSet::new(); + let mut session_ids = BTreeSet::new(); - self.reprice_events_in_tx(&mut report, &mut dirty_keys)?; + self.reprice_events_in_tx(&mut report, &mut dirty_keys, &mut session_ids)?; self.reprice_summaries_in_tx(&mut report)?; report.refreshed_rollups = self.refresh_sync_rollups_for_keys_counted(&dirty_keys)?; + self.refresh_session_rollups_for_keys(&session_ids)?; self.reprice_task_spans_in_tx(&mut report)?; self.set_metadata_value( @@ -185,6 +187,7 @@ impl Store { &self, report: &mut RepricingReport, dirty_keys: &mut BTreeSet, + session_ids: &mut BTreeSet, ) -> Result<()> { let mut after: Option = None; loop { @@ -199,6 +202,7 @@ impl Store { maybe_fail_after_event_writes(report.changed_events)?; if let Some(updated) = reprice_event(&event) { dirty_keys.insert(self.update_event_cost_payload(&updated)?); + session_ids.insert(updated.session.session_id.clone()); report.changed_events += 1; } } diff --git a/crates/statsai-store/src/pricing/tests/repricing.rs b/crates/statsai-store/src/pricing/tests/repricing.rs index 7dd18e1..266b670 100644 --- a/crates/statsai-store/src/pricing/tests/repricing.rs +++ b/crates/statsai-store/src/pricing/tests/repricing.rs @@ -384,7 +384,7 @@ fn daily_rollups_are_unused_by_report_and_sync_paths() { assert_eq!(events[0].cost, expected_review_cost(started_at)); // Pin: bump this when CURRENT_SCHEMA_VERSION changes, after confirming // the daily_rollups table is still unused by report/sync/snapshot. - assert_eq!(CURRENT_SCHEMA_VERSION, 25); + assert_eq!(CURRENT_SCHEMA_VERSION, 28); } #[test] diff --git a/crates/statsai-store/src/pricing/tests/support.rs b/crates/statsai-store/src/pricing/tests/support.rs index 7579b66..828b453 100644 --- a/crates/statsai-store/src/pricing/tests/support.rs +++ b/crates/statsai-store/src/pricing/tests/support.rs @@ -70,6 +70,7 @@ pub(super) fn test_event( started_at, ended_at: None, duration_seconds: None, + turn_started_at: None, }, model: Some(test_model(model)), usage, diff --git a/crates/statsai-store/src/quota/tests/support.rs b/crates/statsai-store/src/quota/tests/support.rs index a435293..c4e634d 100644 --- a/crates/statsai-store/src/quota/tests/support.rs +++ b/crates/statsai-store/src/quota/tests/support.rs @@ -139,6 +139,7 @@ pub(super) fn sample_usage_event( started_at, ended_at: None, duration_seconds: None, + turn_started_at: None, }, model: None, usage: UsageCounts { diff --git a/crates/statsai-store/src/rollups/mod.rs b/crates/statsai-store/src/rollups/mod.rs index 7dc846f..38807a1 100644 --- a/crates/statsai-store/src/rollups/mod.rs +++ b/crates/statsai-store/src/rollups/mod.rs @@ -1,8 +1,12 @@ use super::*; mod build; +mod sessions; mod summaries; +pub(crate) use sessions::task_span_session_hashes; +pub use sessions::{SessionFilter, SessionSort, SessionStats}; + pub(crate) use build::*; pub(crate) use summaries::*; diff --git a/crates/statsai-store/src/rollups/sessions.rs b/crates/statsai-store/src/rollups/sessions.rs new file mode 100644 index 0000000..ac6745c --- /dev/null +++ b/crates/statsai-store/src/rollups/sessions.rs @@ -0,0 +1,1244 @@ +use crate::{ + safe_u64_to_i64, sqlite_in_clause_placeholders, sqlite_string_params, Store, + EVENT_CONVERSATION_HASH_SQL, EVENT_SESSION_ID_SQL, +}; +use anyhow::Result; +use chrono::{DateTime, Utc}; +use rusqlite::{params, OptionalExtension}; +use statsai_core::{ + hash_text, project_has_stable_identity, provider_session_name, + provider_session_title_is_unusable, task_title_is_generic, Confidence, CostAccumulator, + CostInfo, ModelInfo, SessionName, SessionRollupV1, SessionTitleSource, SummaryModelUsage, + UsageCounts, UsageEvent, SESSION_ROLLUP_SCHEMA_VERSION, +}; +use std::collections::{BTreeMap, BTreeSet}; + +// v2 drops record-keyed rows and sessions with neither project nor title, +// leaves single-instant durations unknown, and joins Codex rollout-stem span +// titles. Hosted session tables were rebuilt for the same release, so the v2 +// rebuild resends every session. +const SESSION_ROLLUPS_BACKFILL_KEY: &str = "session_rollups_backfilled_v2"; +/// Matches the adapters' `provider_record_usage_event.v1` event key version. +const PROVIDER_RECORD_ID_PREFIX: &str = "provider_record_usage_event.v1:"; +const UNASSIGNED_ACCOUNT: &str = "unassigned"; + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SessionFilter { + pub provider: Option, + pub project_key: Option, + pub account: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum SessionSort { + #[default] + Started, + Tokens, + Cost, + Duration, + Messages, +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +pub struct SessionStats { + pub sessions: u64, + pub total_tokens: u64, + pub total_cost_micro_usd: i64, + pub avg_tokens: Option, + pub median_tokens: Option, + pub avg_messages: Option, + pub median_duration_seconds: Option, + pub median_active_seconds: Option, + pub total_messages: u64, + pub top_model: Option, +} + +impl Store { + pub(crate) fn backfill_session_rollups_if_needed(&self) -> Result<()> { + if self + .metadata_value(SESSION_ROLLUPS_BACKFILL_KEY)? + .as_deref() + == Some("1") + { + return Ok(()); + } + self.rebuild_session_rollups()?; + // Hosted session tables were rebuilt empty for this release. Resends + // follow each target's acknowledged payload hash, so clearing the hash + // makes the next sync send every session; where a hosted row survived, + // that upsert writes nothing. The acknowledgements themselves stay, so + // a session the rebuild dropped is still retired by the next snapshot. + self.conn.execute( + "UPDATE entity_sync_state SET payload_hash = '' WHERE entity_kind = 'session_rollup'", + [], + )?; + self.set_metadata_value(SESSION_ROLLUPS_BACKFILL_KEY, "1")?; + Ok(()) + } + + pub fn rebuild_session_rollups(&self) -> Result { + let events = self.events()?; + let mut grouped: BTreeMap> = BTreeMap::new(); + for event in events { + if event_has_own_record_session(&event) { + continue; + } + grouped + .entry(event.session.session_id.clone()) + .or_default() + .push(event); + } + self.with_immediate_transaction(|| { + // Every session is resent after a rebuild, since the hosted table + // may have been rebuilt with it. Unchanged rows keep their payload + // and updated_at, so where the hosted row survived, the resend is + // a no-op upsert that writes nothing. + let existing = self.session_rollup_payload_hashes()?; + let hashes = grouped + .values() + .flatten() + .filter_map(|event| event.session.local_session_id_hash.clone()) + .collect::>(); + let titles = self.session_titles_for_hashes(&hashes)?; + let updated_at = Utc::now(); + let rollups = grouped + .values() + .map(|events| build_session_rollup(events, &titles, updated_at)) + .filter(session_rollup_is_identifiable) + .collect::>(); + let kept = rollups + .iter() + .map(|rollup| rollup.session_id.as_str()) + .collect::>(); + for session_id in existing.keys() { + if !kept.contains(session_id.as_str()) { + self.conn.execute( + "DELETE FROM session_rollups WHERE session_id = ?1", + params![session_id], + )?; + } + } + for rollup in &rollups { + let session_id = &rollup.session_id; + let payload_hash = session_rollup_content_hash(rollup)?; + if existing.get(session_id) == Some(&payload_hash) { + self.conn.execute( + "UPDATE session_rollups SET dirty = 1 WHERE session_id = ?1", + params![session_id], + )?; + continue; + } + self.write_session_rollup(rollup, &payload_hash, 1)?; + } + Ok(rollups.len() as u64) + }) + } + + pub(crate) fn refresh_session_rollups_for_keys( + &self, + session_ids: &BTreeSet, + ) -> Result<()> { + if session_ids.is_empty() { + return Ok(()); + } + let mut present = BTreeMap::new(); + let mut hashes = Vec::new(); + for session_id in session_ids { + let events = self.session_events(session_id)?; + if events.is_empty() { + self.conn.execute( + "DELETE FROM session_rollups WHERE session_id = ?1", + params![session_id], + )?; + continue; + } + hashes.extend( + events + .iter() + .filter_map(|event| event.session.local_session_id_hash.clone()), + ); + present.insert(session_id.clone(), events); + } + // Title resolution scans task spans and archives once per batch. Doing + // it per session turns a scan into one full pass for every conversation. + let titles = self.session_titles_for_hashes(&hashes)?; + for (session_id, events) in present { + let rollup = build_session_rollup(&events, &titles, Utc::now()); + if !session_rollup_is_identifiable(&rollup) { + self.conn.execute( + "DELETE FROM session_rollups WHERE session_id = ?1", + params![session_id], + )?; + continue; + } + let payload_hash = session_rollup_content_hash(&rollup)?; + let existing: Option = self + .conn + .query_row( + "SELECT payload_hash FROM session_rollups WHERE session_id = ?1", + params![session_id], + |row| row.get(0), + ) + .optional()?; + if existing.as_deref() == Some(payload_hash.as_str()) { + continue; + } + self.write_session_rollup(&rollup, &payload_hash, 1)?; + } + Ok(()) + } + + /// Records the names providers hold for sessions and refreshes the sessions + /// whose name changed, with the sub-agents that borrow it. Unchanged names + /// write nothing. A session's own name is never replaced by a borrowed one. + pub fn upsert_session_names(&self, names: &[SessionName]) -> Result { + if names.is_empty() { + return Ok(0); + } + let mut changed_hashes = Vec::new(); + { + let mut statement = self.conn.prepare( + r#" + INSERT INTO session_names (local_session_id_hash, title, parent_hash, updated_at) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(local_session_id_hash) DO UPDATE SET + title = excluded.title, + parent_hash = excluded.parent_hash, + updated_at = excluded.updated_at + WHERE (session_names.title IS NOT excluded.title + OR session_names.parent_hash IS NOT excluded.parent_hash) + AND (excluded.parent_hash IS NULL OR session_names.parent_hash IS NOT NULL) + "#, + )?; + let now = Utc::now().to_rfc3339(); + for name in names { + if statement.execute(params![ + &name.local_session_id_hash, + &name.title, + &name.parent_local_session_id_hash, + &now + ])? > 0 + { + changed_hashes.push(name.local_session_id_hash.clone()); + } + } + } + for chunk in changed_hashes.clone().chunks(500) { + let placeholders = sqlite_in_clause_placeholders(chunk.len()); + let mut statement = self.conn.prepare(&format!( + "SELECT local_session_id_hash FROM session_names WHERE parent_hash IN ({placeholders})" + ))?; + let params = sqlite_string_params(chunk); + let rows = statement.query_map(params.as_slice(), |row| row.get::<_, String>(0))?; + for row in rows { + changed_hashes.push(row?); + } + } + let session_ids = self.session_ids_for_local_hashes(&changed_hashes)?; + self.refresh_session_rollups_for_keys(&session_ids)?; + Ok(changed_hashes.len() as u64) + } + + pub(crate) fn refresh_session_rollups_for_raw_ids(&self, raw_ids: &[String]) -> Result<()> { + let hashes = raw_ids + .iter() + .map(|raw_id| raw_id.trim()) + .filter(|raw_id| !raw_id.is_empty()) + .flat_map(|raw_id| std::iter::once(raw_id).chain(codex_rollout_session_uuid(raw_id))) + .map(hash_text) + .collect::>(); + let session_ids = self.session_ids_for_local_hashes(&hashes)?; + self.refresh_session_rollups_for_keys(&session_ids) + } + + fn write_session_rollup( + &self, + rollup: &SessionRollupV1, + payload_hash: &str, + dirty: i64, + ) -> Result<()> { + let payload = serde_json::to_string(rollup)?; + let day_key = rollup + .started_at + .date_naive() + .format("%Y-%m-%d") + .to_string(); + self.conn.execute( + r#" + INSERT INTO session_rollups ( + session_id, provider, source_id, provider_account_id, day_key, + started_at, updated_at, total_tokens, payload_hash, dirty, payload + ) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) + ON CONFLICT(session_id) DO UPDATE SET + provider = excluded.provider, + source_id = excluded.source_id, + provider_account_id = excluded.provider_account_id, + day_key = excluded.day_key, + started_at = excluded.started_at, + updated_at = excluded.updated_at, + total_tokens = excluded.total_tokens, + payload_hash = excluded.payload_hash, + dirty = excluded.dirty, + payload = excluded.payload + "#, + params![ + &rollup.session_id, + &rollup.provider, + &rollup.source_id.0, + rollup.provider_account_id.as_ref().map(|id| id.0.as_str()), + &day_key, + rollup.started_at.to_rfc3339(), + rollup.updated_at.to_rfc3339(), + safe_u64_to_i64(rollup.usage.computed_total()), + payload_hash, + dirty, + &payload, + ], + )?; + Ok(()) + } + + pub fn all_session_rollups(&self) -> Result> { + self.session_rollups_by_sql( + "SELECT payload FROM session_rollups ORDER BY started_at, session_id", + ) + } + + pub fn session_rollup_ids(&self) -> Result> { + let mut statement = self + .conn + .prepare("SELECT session_id FROM session_rollups ORDER BY session_id")?; + let rows = statement.query_map([], |row| row.get::<_, String>(0))?; + rows.collect::, _>>().map_err(Into::into) + } + + pub fn dirty_session_rollups(&self) -> Result> { + self.session_rollups_by_sql( + "SELECT payload FROM session_rollups WHERE dirty = 1 ORDER BY started_at, session_id", + ) + } + + pub fn mark_session_rollups_synced(&self, session_ids: &[String]) -> Result<()> { + if session_ids.is_empty() { + return Ok(()); + } + self.with_immediate_transaction(|| { + self.mark_session_rollups_synced_in_transaction(session_ids) + }) + } + + pub(crate) fn mark_session_rollups_synced_in_transaction( + &self, + session_ids: &[String], + ) -> Result<()> { + for session_id in session_ids { + self.conn.execute( + "UPDATE session_rollups SET dirty = 0 WHERE session_id = ?1", + params![session_id], + )?; + } + Ok(()) + } + + pub fn mark_all_session_rollups_dirty(&self) -> Result { + let updated = self.conn.execute( + "UPDATE session_rollups SET dirty = 1, updated_at = ?1", + params![Utc::now().to_rfc3339()], + )? as u64; + Ok(updated) + } + + pub fn session_rollups_in_period( + &self, + since: Option>, + until: DateTime, + filter: &SessionFilter, + sort: SessionSort, + limit: usize, + offset: usize, + ) -> Result> { + let mut rollups = self.filtered_session_rollups(since, until, filter)?; + sort_sessions(&mut rollups, sort); + Ok(rollups.into_iter().skip(offset).take(limit).collect()) + } + + pub fn session_stats_in_period( + &self, + since: Option>, + until: DateTime, + filter: &SessionFilter, + ) -> Result { + let rollups = self.filtered_session_rollups(since, until, filter)?; + Ok(session_stats(&rollups)) + } + + fn filtered_session_rollups( + &self, + since: Option>, + until: DateTime, + filter: &SessionFilter, + ) -> Result> { + let since = since.map(|value| value.to_rfc3339()); + let until = until.to_rfc3339(); + let provider = filter.provider.as_deref(); + let account = filter.account.as_deref(); + let mut statement = self.conn.prepare( + r#" + SELECT payload FROM session_rollups + WHERE started_at <= ?1 + AND (?2 IS NULL OR started_at >= ?2) + AND (?3 IS NULL OR provider = ?3) + AND ( + ?4 IS NULL + OR (?4 = ?5 AND provider_account_id IS NULL) + OR provider_account_id = ?4 + ) + ORDER BY started_at, session_id + "#, + )?; + let rows = statement.query_map( + params![until, since, provider, account, UNASSIGNED_ACCOUNT], + |row| row.get::<_, String>(0), + )?; + let mut rollups = Vec::new(); + for row in rows { + let rollup: SessionRollupV1 = serde_json::from_str(&row?)?; + if session_matches_project(&rollup, filter.project_key.as_deref()) { + rollups.push(rollup); + } + } + Ok(rollups) + } + + fn session_rollups_by_sql(&self, sql: &str) -> Result> { + let mut statement = self.conn.prepare(sql)?; + let rows = statement.query_map([], |row| row.get::<_, String>(0))?; + let mut rollups = Vec::new(); + for row in rows { + rollups.push(serde_json::from_str(&row?)?); + } + Ok(rollups) + } + + fn session_events(&self, session_id: &str) -> Result> { + let sql = format!( + "SELECT payload FROM usage_events WHERE {EVENT_SESSION_ID_SQL} = ?1 ORDER BY started_at, event_id" + ); + let mut statement = self.conn.prepare(&sql)?; + let rows = statement.query_map(params![session_id], |row| row.get::<_, String>(0))?; + let mut events = Vec::new(); + for row in rows { + let event: UsageEvent = serde_json::from_str(&row?)?; + if !event_has_own_record_session(&event) { + events.push(event); + } + } + Ok(events) + } + + fn session_rollup_payload_hashes(&self) -> Result> { + let mut statement = self + .conn + .prepare("SELECT session_id, payload_hash FROM session_rollups")?; + let rows = statement.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })?; + Ok(rows.collect::, _>>()?) + } + + fn session_ids_for_local_hashes(&self, hashes: &[String]) -> Result> { + let mut session_ids = BTreeSet::new(); + if hashes.is_empty() { + return Ok(session_ids); + } + for chunk in hashes.chunks(500) { + let placeholders = sqlite_in_clause_placeholders(chunk.len()); + let sql = format!( + "SELECT DISTINCT json_extract(payload, '$.session.session_id') + FROM usage_events + WHERE {EVENT_CONVERSATION_HASH_SQL} IN ({placeholders}) + AND json_extract(payload, '$.session.session_id') IS NOT NULL" + ); + let mut statement = self.conn.prepare(&sql)?; + let params = sqlite_string_params(chunk); + let rows = statement.query_map(params.as_slice(), |row| row.get::<_, String>(0))?; + for row in rows { + session_ids.insert(row?); + } + } + Ok(session_ids) + } + + /// Titles keyed by `local_session_id_hash`. Task spans replace archive titles. + /// Generic and secret-looking titles are dropped. + pub(crate) fn session_titles_for_hashes( + &self, + hashes: &[String], + ) -> Result> { + let wanted = hashes + .iter() + .map(String::as_str) + .collect::>() + .into_iter() + .collect::>(); + let mut titles = BTreeMap::new(); + if wanted.is_empty() { + return Ok(titles); + } + // Rows are gathered across chunks and then ordered, so the latest title + // wins no matter which chunk held it. + let mut archive_rows = Vec::new(); + let mut span_rows = Vec::new(); + let mut name_rows = Vec::new(); + for chunk in wanted.chunks(300) { + let placeholders = sqlite_in_clause_placeholders(chunk.len()); + let params = chunk + .iter() + .map(|hash| hash as &dyn rusqlite::types::ToSql) + .collect::>(); + let mut archive = self.conn.prepare(&format!( + "SELECT updated_at, conversation_id, session_hash, title + FROM archive_conversations + WHERE session_hash IN ({placeholders}) AND title IS NOT NULL" + ))?; + for row in archive.query_map(params.as_slice(), |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + )) + })? { + archive_rows.push(row?); + } + // A Codex span may name its session by rollout path; the events + // carry the thread id inside it. + let mut spans = self.conn.prepare(&format!( + "SELECT started_at, span_id, + CASE WHEN session_hash IN ({placeholders}) THEN session_hash + ELSE session_uuid_hash END, + title + FROM task_spans + WHERE is_meta = 0 + AND (session_hash IN ({placeholders}) + OR session_uuid_hash IN ({placeholders}))" + ))?; + let mut span_params = params.clone(); + span_params.extend(params.iter().copied()); + span_params.extend(params.iter().copied()); + for row in spans.query_map(span_params.as_slice(), |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, Option>(3)?, + )) + })? { + span_rows.push(row?); + } + // A borrowed name reads the parent's own name as it is now. + let mut names = self.conn.prepare(&format!( + "SELECT child.local_session_id_hash, + CASE WHEN child.parent_hash IS NULL THEN child.title + WHEN parent.title IS NOT NULL + THEN parent.title || ' · ' || child.title + END + FROM session_names child + LEFT JOIN session_names parent + ON parent.local_session_id_hash = child.parent_hash + AND parent.parent_hash IS NULL + WHERE child.local_session_id_hash IN ({placeholders})" + ))?; + for row in names.query_map(params.as_slice(), |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)) + })? { + name_rows.push(row?); + } + } + archive_rows.sort(); + for (_, _, hash, title) in archive_rows { + remember_session_title(&mut titles, hash, Some(&title), SessionTitleSource::Archive); + } + span_rows.sort(); + for (_, _, hash, title) in span_rows { + remember_session_title( + &mut titles, + hash, + title.as_deref(), + SessionTitleSource::TaskSpan, + ); + } + // A provider-held name outranks task and archive titles for its session. + for (hash, title) in name_rows { + let title = provider_session_name(title.as_deref(), 90); + remember_session_title( + &mut titles, + hash, + title.as_deref(), + SessionTitleSource::Event, + ); + } + Ok(titles) + } + + pub(crate) fn task_span_raw_session_ids_for_bucket( + &self, + project_bucket: &str, + ) -> Result> { + let mut statement = self.conn.prepare( + "SELECT json_extract(payload, '$.session_id') + FROM task_spans + WHERE project_bucket = ?1", + )?; + let rows = statement.query_map(params![project_bucket], |row| { + row.get::<_, Option>(0) + })?; + let mut ids = BTreeSet::new(); + for row in rows { + if let Some(session_id) = row?.filter(|value| !value.trim().is_empty()) { + ids.insert(session_id); + } + } + Ok(ids) + } +} + +pub(crate) fn build_session_rollup( + events: &[UsageEvent], + titles: &BTreeMap, + updated_at: DateTime, +) -> SessionRollupV1 { + let mut ordered = events.to_vec(); + ordered.sort_by(|left, right| { + left.session + .started_at + .cmp(&right.session.started_at) + .then(left.created_at.cmp(&right.created_at)) + .then(left.event_id.0.cmp(&right.event_id.0)) + }); + let events = ordered.as_slice(); + let oldest = events.first().expect("session rollup requires events"); + let newest = events.last().unwrap_or(oldest); + let mut totals = UsageTotals::default(); + let mut estimated_cost = CostAccumulator::default(); + let mut provider_reported_cost = CostAccumulator::default(); + let mut has_provider_reported = false; + let mut model_buckets: BTreeMap = BTreeMap::new(); + let mut total_messages = 0u64; + let mut user_messages = 0u64; + let mut assistant_messages = 0u64; + let mut developer_messages = 0u64; + let mut saw_messages = false; + + for event in events { + totals.add(&event.usage); + let requests = event.usage.requests.unwrap_or(1); + totals.requests = totals.requests.saturating_add(requests); + estimated_cost.add_estimated(&event.cost); + if event.cost.provider_reported_micro_usd.is_some() + || event.cost.provider_reported_usd.is_some() + { + provider_reported_cost.add_values( + event.cost.provider_reported_micro_usd, + event.cost.provider_reported_usd, + ); + has_provider_reported = true; + } + if let Some(runtime) = event.runtime.as_ref() { + let derived_total = runtime.total_messages.or_else(|| { + let derived = runtime + .user_messages + .unwrap_or(0) + .saturating_add(runtime.assistant_messages.unwrap_or(0)) + .saturating_add(runtime.developer_messages.unwrap_or(0)); + (derived > 0).then_some(derived) + }); + if derived_total.is_some() + || runtime.user_messages.is_some() + || runtime.assistant_messages.is_some() + || runtime.developer_messages.is_some() + { + saw_messages = true; + } + total_messages = total_messages.saturating_add(derived_total.unwrap_or(0)); + user_messages = user_messages.saturating_add(runtime.user_messages.unwrap_or(0)); + assistant_messages = + assistant_messages.saturating_add(runtime.assistant_messages.unwrap_or(0)); + developer_messages = + developer_messages.saturating_add(runtime.developer_messages.unwrap_or(0)); + } + + let model = event.model.clone().unwrap_or_default(); + let entry = model_buckets + .entry(session_model_key(&model)) + .or_insert_with(|| (model, ModelTotals::default())); + entry.1.add(&event.usage, requests, &event.cost); + } + + // A turn starts at its prompt, which can precede the turn's first event. + let started_at = events + .iter() + .map(|event| { + event + .session + .turn_started_at + .map_or(event.session.started_at, |turn| { + turn.min(event.session.started_at) + }) + }) + .min() + .unwrap_or(oldest.session.started_at); + let mut ended_at = events + .iter() + .map(|event| event.session.ended_at.unwrap_or(event.created_at)) + .max() + .unwrap_or(newest.created_at); + // A provider clock can record a completion before the session start. + // Ordering the interval keeps duration non-negative. + if ended_at < started_at { + ended_at = started_at; + } + // One instant with no recorded end is a start, not a zero-length session. + // Counting it as 0s pulled the median to zero for every source that + // stamps a single time per row. + let has_recorded_end = events + .iter() + .any(|event| event.session.ended_at.is_some() || event.session.duration_seconds.is_some()); + let duration_seconds = if has_recorded_end || ended_at > started_at { + ended_at + .signed_duration_since(started_at) + .num_seconds() + .try_into() + .ok() + } else { + None + }; + let active_seconds = session_active_seconds(events); + let provider_account_id = events + .iter() + .rev() + .find_map(|event| event.provider_account_id.clone()); + // The latest event that knows its project: a later event without project + // metadata must not erase it, since a session with neither a project nor + // a title is not kept. + let project = events + .iter() + .rev() + .find_map(|event| event.project.clone().filter(project_has_stable_identity)); + let (title, title_source) = resolve_title(events, titles); + let models = model_buckets + .into_iter() + .map(|(_, (model, totals))| model_usage(model, totals)) + .collect::>(); + let primary_model = primary_model_label(&models); + let requests = totals.requests; + + SessionRollupV1 { + schema_version: SESSION_ROLLUP_SCHEMA_VERSION.to_string(), + session_id: oldest.session.session_id.clone(), + device_id: newest.device_id.clone(), + provider: newest.provider.clone(), + source_id: newest.source_id.clone(), + provider_account_id, + started_at, + ended_at, + duration_seconds, + active_seconds, + usage: totals.into_counts(), + requests, + cost: CostInfo { + currency: "USD".to_string(), + estimated_api_equivalent_usd: estimated_cost.cents_rounded(), + provider_reported_usd: has_provider_reported + .then(|| provider_reported_cost.cents_rounded()) + .flatten(), + estimated_api_equivalent_micro_usd: estimated_cost.micro_usd(), + provider_reported_micro_usd: has_provider_reported + .then(|| provider_reported_cost.micro_usd()) + .flatten(), + pricing_source: Some("local_rollup".to_string()), + pricing_version: None, + confidence: Confidence::Medium, + }, + models, + primary_model, + total_messages: saw_messages + .then_some(total_messages) + .filter(|value| *value > 0), + user_messages: (user_messages > 0).then_some(user_messages), + assistant_messages: (assistant_messages > 0).then_some(assistant_messages), + developer_messages: (developer_messages > 0).then_some(developer_messages), + project, + title, + title_source, + updated_at, + } +} + +/// Time the agent was working: the union of the intervals the provider +/// recorded for its turns. Codex records a turn's start and completion, +/// OpenCode each message's, and Claude Code events carry the prompt that +/// started their turn. Gaps between turns are idle and do not count; nothing +/// here guesses where a session went idle. +/// +/// An event that aggregates a whole session (OpenCode's `sqlite:session` +/// fallback) carries the session's lifetime, which is a span, not a turn. +fn session_active_seconds(events: &[UsageEvent]) -> Option { + let mut intervals = events + .iter() + .filter(|event| !event.source.source_type.ends_with(":session")) + .filter_map(|event| { + let session = &event.session; + let (start, mut end) = match session.turn_started_at { + Some(turn) => (turn, session.ended_at.unwrap_or(session.started_at)), + None => (session.started_at, session.ended_at?), + }; + // A reported duration wins over the completion time. Codex writes a + // turn's completion when the thread next runs, so a turn that + // worked 21 minutes can complete four days after it started. + if let Some(reported) = session + .duration_seconds + .and_then(|seconds| i64::try_from(seconds).ok()) + { + end = end.min(start + chrono::Duration::seconds(reported)); + } + (end >= start).then_some((start, end)) + }) + .collect::>(); + intervals.sort(); + let mut intervals = intervals.into_iter(); + let (mut start, mut end) = intervals.next()?; + let mut total = chrono::Duration::zero(); + for (next_start, next_end) in intervals { + if next_start <= end { + end = end.max(next_end); + } else { + total += end - start; + (start, end) = (next_start, next_end); + } + } + total += end - start; + u64::try_from(total.num_seconds()).ok() +} + +/// Precedence: the name the provider holds now, then a title stamped on an +/// event when it was parsed, then a task-span title, then an archive title. +fn resolve_title( + events: &[UsageEvent], + titles: &BTreeMap, +) -> (Option, Option) { + let hashes = || { + events + .iter() + .rev() + .filter_map(|event| event.session.local_session_id_hash.as_deref()) + }; + if let Some(title) = hashes().find_map(|hash| { + titles + .get(hash) + .filter(|(_, source)| *source == SessionTitleSource::Event) + .map(|(title, _)| title.clone()) + }) { + return (Some(title), Some(SessionTitleSource::Event)); + } + if let Some(title) = events.iter().rev().find_map(|event| { + acceptable_session_title(event.session.title.as_deref(), SessionTitleSource::Event) + }) { + return (Some(title), Some(SessionTitleSource::Event)); + } + for hash in hashes() { + if let Some((title, source)) = titles.get(hash) { + return (Some(title.clone()), Some(*source)); + } + } + (None, None) +} + +/// A session with neither a project nor a title cannot be told apart from any +/// other, so it is not kept. Cursor cloud agents are the case: the export names +/// the agent by id only and records no project. +fn session_rollup_is_identifiable(rollup: &SessionRollupV1) -> bool { + rollup.project.is_some() || rollup.title.is_some() +} + +/// An event whose session identity is its own provider record belongs to no +/// session. Cursor's usage export names no session, so each local row is keyed +/// by itself, and treating those rows as sessions listed every request as one. +pub(crate) fn event_has_own_record_session(event: &UsageEvent) -> bool { + let Some(session_hash) = event.session.local_session_id_hash.as_deref() else { + return false; + }; + event + .parse_evidence + .as_ref() + .and_then(|evidence| evidence.source_record_id.as_deref()) + .and_then(|record_id| record_id.strip_prefix(PROVIDER_RECORD_ID_PREFIX)) + .and_then(|rest| rest.rsplit_once(':')) + .is_some_and(|(_, record_hash)| record_hash == session_hash) +} + +/// Older Codex task spans name their session by the rollout file stem, while +/// events hash the UUID the file declares. The stem ends in that UUID. +fn codex_rollout_session_uuid(raw_id: &str) -> Option<&str> { + let stem = raw_id.rsplit('/').next()?; + if !stem.starts_with("rollout-") { + return None; + } + let uuid = stem.get(stem.len().checked_sub(36)?..)?; + let is_uuid = uuid.char_indices().all(|(index, character)| match index { + 8 | 13 | 18 | 23 => character == '-', + _ => character.is_ascii_hexdigit(), + }); + is_uuid.then_some(uuid) +} + +/// Names the provider put on its events are kept unless unsafe. Task-span and +/// archive titles can come from prompts, so they must not be generic. +fn acceptable_session_title(value: Option<&str>, source: SessionTitleSource) -> Option { + let trimmed = value?.trim(); + let rejected = match source { + SessionTitleSource::Event => provider_session_title_is_unusable(Some(trimmed)), + _ => task_title_is_generic(Some(trimmed)), + }; + (!trimmed.is_empty() && !rejected).then(|| trimmed.to_string()) +} + +fn remember_session_title( + titles: &mut BTreeMap, + hash: String, + title: Option<&str>, + source: SessionTitleSource, +) { + if let Some(title) = acceptable_session_title(title, source) { + titles.insert(hash, (title, source)); + } +} + +/// The hashed keys a task span's session is looked up by: its session id, and +/// for a Codex rollout path, the thread id inside it. +pub(crate) fn task_span_session_hashes( + session_id: Option<&str>, +) -> (Option, Option) { + let Some(session_id) = session_id.filter(|value| !value.trim().is_empty()) else { + return (None, None); + }; + ( + Some(hash_text(session_id)), + codex_rollout_session_uuid(session_id).map(hash_text), + ) +} + +fn session_rollup_content_hash(rollup: &SessionRollupV1) -> Result { + let mut comparable = rollup.clone(); + comparable.updated_at = DateTime::::UNIX_EPOCH; + Ok(hash_text(&serde_json::to_string(&comparable)?)) +} + +fn session_matches_project(rollup: &SessionRollupV1, project: Option<&str>) -> bool { + let Some(project) = project.map(str::trim).filter(|value| !value.is_empty()) else { + return true; + }; + let Some(info) = rollup.project.as_ref() else { + return false; + }; + if statsai_core::daily_rollup_project_key(Some(info)) == project || info.project_id == project { + return true; + } + [ + info.project_label.as_deref(), + info.repo_label.as_deref(), + info.path_label.as_deref(), + ] + .into_iter() + .flatten() + .any(|label| label.eq_ignore_ascii_case(project)) +} + +fn sort_sessions(sessions: &mut [SessionRollupV1], sort: SessionSort) { + sessions.sort_by(|left, right| { + let metric = match sort { + SessionSort::Started => std::cmp::Ordering::Equal, + SessionSort::Tokens => right + .usage + .computed_total() + .cmp(&left.usage.computed_total()), + SessionSort::Cost => right + .cost + .estimated_micro_usd() + .unwrap_or(0) + .cmp(&left.cost.estimated_micro_usd().unwrap_or(0)), + // Active time: a thread resumed weeks later is not a long session. + SessionSort::Duration => right + .active_seconds + .unwrap_or(0) + .cmp(&left.active_seconds.unwrap_or(0)), + SessionSort::Messages => right + .total_messages + .unwrap_or(0) + .cmp(&left.total_messages.unwrap_or(0)), + }; + metric + .then_with(|| right.started_at.cmp(&left.started_at)) + .then_with(|| left.session_id.cmp(&right.session_id)) + }); +} + +fn session_stats(rollups: &[SessionRollupV1]) -> SessionStats { + let sessions = rollups.len() as u64; + let total_tokens = rollups.iter().fold(0u64, |sum, rollup| { + sum.saturating_add(rollup.usage.computed_total()) + }); + let total_cost_micro_usd = rollups.iter().fold(0i64, |sum, rollup| { + sum.saturating_add(rollup.cost.estimated_micro_usd().unwrap_or(0)) + }); + let total_messages = rollups.iter().fold(0u64, |sum, rollup| { + sum.saturating_add(rollup.total_messages.unwrap_or(0)) + }); + let message_samples = rollups + .iter() + .filter_map(|rollup| rollup.total_messages) + .collect::>(); + let avg_messages = (!message_samples.is_empty()) + .then(|| message_samples.iter().sum::() as f64 / message_samples.len() as f64); + let mut token_samples = rollups + .iter() + .map(|rollup| rollup.usage.computed_total()) + .collect::>(); + let median_tokens = median_u64(&mut token_samples); + let mut durations = rollups + .iter() + .filter_map(|rollup| rollup.duration_seconds) + .collect::>(); + let mut active = rollups + .iter() + .filter_map(|rollup| rollup.active_seconds) + .collect::>(); + // Each session's per-model usage, not its primary model's whole total: a + // session that is 60% one model and 40% another counts both. A session + // without per-model usage counts its primary model. + let mut model_tokens: BTreeMap = BTreeMap::new(); + for rollup in rollups { + let mut counted = false; + for usage in &rollup.models { + if let Some(label) = model_label(&usage.model) { + let entry = model_tokens.entry(label).or_default(); + *entry = (*entry).saturating_add(usage.usage.computed_total()); + counted = true; + } + } + if !counted { + if let Some(model) = rollup.primary_model.clone() { + let entry = model_tokens.entry(model).or_default(); + *entry = (*entry).saturating_add(rollup.usage.computed_total()); + } + } + } + let top_model = model_tokens + .into_iter() + .max_by(|left, right| left.1.cmp(&right.1).then_with(|| right.0.cmp(&left.0))) + .map(|(model, _)| model); + SessionStats { + sessions, + total_tokens, + total_cost_micro_usd, + // Lazily: `then_some` would divide by zero when nothing matches. + avg_tokens: (sessions > 0).then(|| total_tokens / sessions), + median_tokens, + avg_messages, + median_duration_seconds: median_u64(&mut durations), + median_active_seconds: median_u64(&mut active), + total_messages, + top_model, + } +} + +fn median_u64(values: &mut [u64]) -> Option { + if values.is_empty() { + return None; + } + values.sort_unstable(); + let mid = values.len() / 2; + if values.len() % 2 == 1 { + Some(values[mid]) + } else { + Some(values[mid - 1].saturating_add(values[mid]) / 2) + } +} + +fn primary_model_label(models: &[SummaryModelUsage]) -> Option { + models + .iter() + .filter_map(|usage| { + model_label(&usage.model).map(|label| (label, usage.usage.computed_total())) + }) + .max_by(|left, right| left.1.cmp(&right.1).then_with(|| right.0.cmp(&left.0))) + .map(|(label, _)| label) +} + +fn model_label(model: &ModelInfo) -> Option { + model + .normalized_name + .clone() + .or(model.name.clone()) + .or(model.provider_model_id.clone()) + .filter(|value| !value.trim().is_empty()) +} + +fn model_usage(model: ModelInfo, totals: ModelTotals) -> SummaryModelUsage { + SummaryModelUsage { + model, + usage: UsageCounts { + input_tokens: Some(totals.input_tokens), + output_tokens: Some(totals.output_tokens), + cache_creation_tokens: Some(totals.cache_creation_tokens), + cache_creation_5m_tokens: Some(totals.cache_creation_5m_tokens), + cache_creation_1h_tokens: Some(totals.cache_creation_1h_tokens), + cache_read_tokens: Some(totals.cache_read_tokens), + reasoning_tokens: Some(totals.reasoning_tokens), + total_tokens: Some(totals.total_tokens), + requests: Some(totals.requests), + local_prompt_eval_tokens: None, + local_eval_tokens: None, + }, + cost: CostInfo { + currency: "USD".to_string(), + estimated_api_equivalent_usd: totals.estimated_cost.cents_rounded(), + provider_reported_usd: totals + .has_provider_reported + .then(|| totals.provider_reported_cost.cents_rounded()) + .flatten(), + estimated_api_equivalent_micro_usd: totals.estimated_cost.micro_usd(), + provider_reported_micro_usd: totals + .has_provider_reported + .then(|| totals.provider_reported_cost.micro_usd()) + .flatten(), + pricing_source: Some("local_rollup".to_string()), + pricing_version: None, + confidence: Confidence::Medium, + }, + metrics: None, + } +} + +fn session_model_key(model: &ModelInfo) -> String { + format!( + "{}|{}|{}|{}|{}|{}", + model.normalized_name.as_deref().unwrap_or(""), + model.provider_model_id.as_deref().unwrap_or(""), + model.name.as_deref().unwrap_or(""), + model.speed.as_deref().unwrap_or(""), + model + .reasoning_level + .as_ref() + .map(|level| level.as_str()) + .unwrap_or(""), + model.reasoning_level_raw.as_deref().unwrap_or("") + ) +} + +#[derive(Debug, Default)] +struct UsageTotals { + input_tokens: u64, + output_tokens: u64, + cache_creation_tokens: u64, + cache_creation_5m_tokens: u64, + cache_creation_1h_tokens: u64, + cache_read_tokens: u64, + reasoning_tokens: u64, + total_tokens: u64, + requests: u64, +} + +impl UsageTotals { + fn add(&mut self, usage: &UsageCounts) { + self.input_tokens = self + .input_tokens + .saturating_add(usage.input_tokens.unwrap_or(0)); + self.output_tokens = self + .output_tokens + .saturating_add(usage.output_tokens.unwrap_or(0)); + self.cache_creation_tokens = self + .cache_creation_tokens + .saturating_add(usage.cache_creation_tokens.unwrap_or(0)); + self.cache_creation_5m_tokens = self + .cache_creation_5m_tokens + .saturating_add(usage.cache_creation_5m_tokens.unwrap_or(0)); + self.cache_creation_1h_tokens = self + .cache_creation_1h_tokens + .saturating_add(usage.cache_creation_1h_tokens.unwrap_or(0)); + self.cache_read_tokens = self + .cache_read_tokens + .saturating_add(usage.cache_read_tokens.unwrap_or(0)); + self.reasoning_tokens = self + .reasoning_tokens + .saturating_add(usage.reasoning_tokens.unwrap_or(0)); + self.total_tokens = self.total_tokens.saturating_add(usage.computed_total()); + } + + fn into_counts(self) -> UsageCounts { + UsageCounts { + input_tokens: Some(self.input_tokens), + output_tokens: Some(self.output_tokens), + cache_creation_tokens: Some(self.cache_creation_tokens), + cache_creation_5m_tokens: Some(self.cache_creation_5m_tokens), + cache_creation_1h_tokens: Some(self.cache_creation_1h_tokens), + cache_read_tokens: Some(self.cache_read_tokens), + reasoning_tokens: Some(self.reasoning_tokens), + total_tokens: Some(self.total_tokens), + requests: Some(self.requests), + local_prompt_eval_tokens: None, + local_eval_tokens: None, + } + } +} + +#[derive(Debug, Default)] +struct ModelTotals { + input_tokens: u64, + output_tokens: u64, + cache_creation_tokens: u64, + cache_creation_5m_tokens: u64, + cache_creation_1h_tokens: u64, + cache_read_tokens: u64, + reasoning_tokens: u64, + total_tokens: u64, + requests: u64, + estimated_cost: CostAccumulator, + provider_reported_cost: CostAccumulator, + has_provider_reported: bool, +} + +impl ModelTotals { + fn add(&mut self, usage: &UsageCounts, requests: u64, cost: &CostInfo) { + self.input_tokens = self + .input_tokens + .saturating_add(usage.input_tokens.unwrap_or(0)); + self.output_tokens = self + .output_tokens + .saturating_add(usage.output_tokens.unwrap_or(0)); + self.cache_creation_tokens = self + .cache_creation_tokens + .saturating_add(usage.cache_creation_tokens.unwrap_or(0)); + self.cache_creation_5m_tokens = self + .cache_creation_5m_tokens + .saturating_add(usage.cache_creation_5m_tokens.unwrap_or(0)); + self.cache_creation_1h_tokens = self + .cache_creation_1h_tokens + .saturating_add(usage.cache_creation_1h_tokens.unwrap_or(0)); + self.cache_read_tokens = self + .cache_read_tokens + .saturating_add(usage.cache_read_tokens.unwrap_or(0)); + self.reasoning_tokens = self + .reasoning_tokens + .saturating_add(usage.reasoning_tokens.unwrap_or(0)); + self.total_tokens = self.total_tokens.saturating_add(usage.computed_total()); + self.requests = self.requests.saturating_add(requests); + self.estimated_cost.add_estimated(cost); + if cost.provider_reported_micro_usd.is_some() || cost.provider_reported_usd.is_some() { + self.provider_reported_cost + .add_values(cost.provider_reported_micro_usd, cost.provider_reported_usd); + self.has_provider_reported = true; + } + } +} diff --git a/crates/statsai-store/src/scan_state.rs b/crates/statsai-store/src/scan_state.rs index eaa22cb..7118b05 100644 --- a/crates/statsai-store/src/scan_state.rs +++ b/crates/statsai-store/src/scan_state.rs @@ -255,6 +255,7 @@ impl Store { )?; let inserted_events = self.insert_events(replacement.events)?; let written_summaries = self.upsert_summaries(replacement.summaries)?; + self.upsert_session_names(replacement.session_names)?; // A quota observation can point at an event this just deleted, and the // daemon -- the only caller -- never touches quota rows itself, so nothing // else repairs the link. Run after the insert: a record that survives the diff --git a/crates/statsai-store/src/sync_state/mod.rs b/crates/statsai-store/src/sync_state/mod.rs index 11e8f86..9008f22 100644 --- a/crates/statsai-store/src/sync_state/mod.rs +++ b/crates/statsai-store/src/sync_state/mod.rs @@ -651,6 +651,20 @@ impl Store { .map(|coverage| coverage.coverage_id.clone()) .collect::>(), )?; + self.record_serialized_entities_synced_in_transaction( + sink, + target, + "session_rollup", + &batch.sessions, + |rollup| rollup.session_id.as_str(), + )?; + self.mark_session_rollups_synced_in_transaction( + &batch + .sessions + .iter() + .map(|rollup| rollup.session_id.clone()) + .collect::>(), + )?; self.record_task_bucket_snapshots_synced_in_transaction( sink, target, diff --git a/crates/statsai-store/src/sync_state/pending.rs b/crates/statsai-store/src/sync_state/pending.rs index 3aeb649..8c38c2b 100644 --- a/crates/statsai-store/src/sync_state/pending.rs +++ b/crates/statsai-store/src/sync_state/pending.rs @@ -162,6 +162,21 @@ impl Store { ) } + pub fn pending_session_rollups_for_sync( + &self, + sink: &str, + target: &str, + rollups: &[statsai_core::SessionRollupV1], + ) -> Result> { + self.pending_serialized_entities_for_sync( + sink, + target, + "session_rollup", + rollups, + |rollup| rollup.session_id.as_str(), + ) + } + pub fn pending_activity_coverage_for_sync( &self, sink: &str, @@ -331,6 +346,13 @@ impl Store { .collect::>(), ), ]); + let mut current_ids = current_ids; + if let Some(ids) = snapshot.session_rollup_ids.as_ref() { + current_ids.insert( + "session_rollup", + ids.iter().map(String::as_str).collect::>(), + ); + } let mut statement = self.conn.prepare( r#" SELECT entity_kind, entity_id @@ -340,7 +362,7 @@ impl Store { 'source', 'account', 'source_account_assignment', 'subscription', 'summary', 'code_change_metric', 'quota_cycle_contribution', 'account_plan_observation', 'account_evidence_summary', - 'activity_rollup', 'activity_coverage' + 'activity_rollup', 'activity_coverage', 'session_rollup' ) "#, )?; @@ -461,6 +483,8 @@ impl Store { ¤t_account_evidence_summaries, )?; let current_snapshot = self.current_http_sync_authoritative_snapshot( + target, + include_projects, ¤t_rollups, ¤t_passthrough_summaries, ¤t_code_change_metrics, @@ -503,8 +527,11 @@ impl Store { }) } + #[allow(clippy::too_many_arguments)] pub(crate) fn current_http_sync_authoritative_snapshot( &self, + target: &str, + include_projects: bool, rollups: &[UsageSummary], passthrough_summaries: &[UsageSummary], code_change_metrics: &[CodeChangeMetric], @@ -564,6 +591,45 @@ impl Store { .into_iter() .map(|coverage| coverage.coverage_id) .collect(), + session_rollup_ids: self.authoritative_session_rollup_ids( + "http", + target, + self.sync_preferences()?.include_sessions, + include_projects, + || self.session_rollup_ids(), + )?, }) } + + /// The session ids an authoritative snapshot names for a target. + /// + /// With session sync on, every local session. With projects excluded, an + /// empty list while the target still holds sessions from this device: each + /// carries its project, path, repository and branch, so excluding projects + /// retires them. With only session sync off, nothing, and hosted sessions + /// stay until it is turned back on. + pub fn authoritative_session_rollup_ids( + &self, + sink: &str, + target: &str, + include_sessions: bool, + include_projects: bool, + session_ids: impl FnOnce() -> Result>, + ) -> Result>> { + if include_sessions { + return session_ids().map(Some); + } + if include_projects { + return Ok(None); + } + let tracked = self.conn.query_row( + "SELECT EXISTS( + SELECT 1 FROM entity_sync_state + WHERE sink = ?1 AND target = ?2 AND entity_kind = 'session_rollup' + )", + params![sink, target], + |row| row.get::<_, bool>(0), + )?; + Ok(tracked.then(Vec::new)) + } } diff --git a/crates/statsai-store/src/sync_state/record.rs b/crates/statsai-store/src/sync_state/record.rs index e61a693..facf30b 100644 --- a/crates/statsai-store/src/sync_state/record.rs +++ b/crates/statsai-store/src/sync_state/record.rs @@ -314,6 +314,26 @@ impl Store { }) } + pub fn record_session_rollups_synced( + &self, + sink: &str, + target: &str, + rollups: &[statsai_core::SessionRollupV1], + ) -> Result<()> { + if rollups.is_empty() { + return Ok(()); + } + self.with_immediate_transaction(|| { + self.record_serialized_entities_synced_in_transaction( + sink, + target, + "session_rollup", + rollups, + |rollup| rollup.session_id.as_str(), + ) + }) + } + pub fn record_activity_coverage_synced( &self, sink: &str, diff --git a/crates/statsai-store/src/tasks/buckets.rs b/crates/statsai-store/src/tasks/buckets.rs index 131bfb4..be053f9 100644 --- a/crates/statsai-store/src/tasks/buckets.rs +++ b/crates/statsai-store/src/tasks/buckets.rs @@ -3,8 +3,23 @@ use super::*; impl Store { pub fn replace_task_bucket_snapshot(&self, snapshot: &TaskBucketSnapshot) -> Result<()> { self.with_immediate_transaction(|| { + let mut raw_session_ids = + self.task_span_raw_session_ids_for_bucket(&snapshot.project_bucket)?; self.delete_task_bucket_snapshot_in_tx(&snapshot.project_bucket)?; self.upsert_task_spans_in_tx(&snapshot.spans)?; + for span in &snapshot.spans { + if let Some(session_id) = span + .session_id + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + { + raw_session_ids.insert(session_id.to_string()); + } + } + self.refresh_session_rollups_for_raw_ids( + &raw_session_ids.into_iter().collect::>(), + )?; self.insert_work_items_in_tx(&snapshot.work_items, &snapshot.members)?; let mut changed_buckets = BTreeSet::new(); changed_buckets.insert(snapshot.project_bucket.clone()); diff --git a/crates/statsai-store/src/tasks/spans.rs b/crates/statsai-store/src/tasks/spans.rs index 580b306..8e81dfa 100644 --- a/crates/statsai-store/src/tasks/spans.rs +++ b/crates/statsai-store/src/tasks/spans.rs @@ -82,9 +82,10 @@ impl Store { span_id, provider, source_id, project_bucket, started_at, ended_at, title, normalized_title, is_meta, confidence, source_file_path_hash, event_count, has_usage_evidence, total_messages, user_messages, assistant_messages, - developer_messages, payload + developer_messages, payload, session_hash, session_uuid_hash ) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, + ?19, ?20) ON CONFLICT(span_id) DO UPDATE SET provider = excluded.provider, source_id = excluded.source_id, @@ -102,7 +103,9 @@ impl Store { user_messages = excluded.user_messages, assistant_messages = excluded.assistant_messages, developer_messages = excluded.developer_messages, - payload = excluded.payload + payload = excluded.payload, + session_hash = excluded.session_hash, + session_uuid_hash = excluded.session_uuid_hash "#, )?; let mut delete_links = self @@ -117,6 +120,8 @@ impl Store { )?; for span in spans { let payload = serde_json::to_string(span)?; + let (session_hash, session_uuid_hash) = + crate::task_span_session_hashes(span.session_id.as_deref()); changed += span_stmt.execute(params![ &span.span_id.0, &span.provider, @@ -136,12 +141,22 @@ impl Store { safe_u64_to_i64(span.assistant_messages), safe_u64_to_i64(span.developer_messages), &payload, + session_hash, + session_uuid_hash, ])? as u64; delete_links.execute(params![&span.span_id.0])?; for event_id in &span.linked_event_ids { link_stmt.execute(params![&span.span_id.0, &event_id.0])?; } } + drop(span_stmt); + drop(delete_links); + drop(link_stmt); + let raw_session_ids = spans + .iter() + .filter_map(|span| span.session_id.clone()) + .collect::>(); + self.refresh_session_rollups_for_raw_ids(&raw_session_ids)?; Ok(changed) } @@ -220,13 +235,32 @@ impl Store { let mut delete_spans = self .conn .prepare("DELETE FROM task_spans WHERE span_id = ?1")?; + // A deleted span can have been the source of a session's title, while + // the session's usage survives under another source. + let mut session_of_span = self.conn.prepare( + "SELECT json_extract(payload, '$.session_id') FROM task_spans WHERE span_id = ?1", + )?; + let mut raw_session_ids = Vec::new(); let mut deleted = 0u64; let mut affected_project_buckets = BTreeSet::new(); for target in targets { affected_project_buckets.insert(target.project_bucket.clone()); + if let Some(session_id) = session_of_span + .query_row(params![&target.span_id.0], |row| { + row.get::<_, Option>(0) + }) + .optional()? + .flatten() + { + raw_session_ids.push(session_id); + } delete_links.execute(params![&target.span_id.0])?; deleted += delete_spans.execute(params![&target.span_id.0])? as u64; } + drop(session_of_span); + drop(delete_links); + drop(delete_spans); + self.refresh_session_rollups_for_raw_ids(&raw_session_ids)?; Ok(TaskDeletionImpact { deleted, affected_project_buckets, diff --git a/crates/statsai-store/src/tests/events.rs b/crates/statsai-store/src/tests/events.rs index 8a005d0..eb74fc9 100644 --- a/crates/statsai-store/src/tests/events.rs +++ b/crates/statsai-store/src/tests/events.rs @@ -39,6 +39,7 @@ fn inserts_events_idempotently() { started_at: now, ended_at: None, duration_seconds: None, + turn_started_at: None, }, model: None, usage: UsageCounts { diff --git a/crates/statsai-store/src/tests/rollups/mod.rs b/crates/statsai-store/src/tests/rollups/mod.rs index ec2a2be..5bcff87 100644 --- a/crates/statsai-store/src/tests/rollups/mod.rs +++ b/crates/statsai-store/src/tests/rollups/mod.rs @@ -2,6 +2,7 @@ pub(super) use super::support::*; pub(crate) use super::*; mod build; +mod sessions; #[test] fn renaming_a_repository_keeps_one_rollup_for_the_day() { diff --git a/crates/statsai-store/src/tests/rollups/sessions.rs b/crates/statsai-store/src/tests/rollups/sessions.rs new file mode 100644 index 0000000..ebfe4c0 --- /dev/null +++ b/crates/statsai-store/src/tests/rollups/sessions.rs @@ -0,0 +1,1124 @@ +use super::*; +use statsai_core::{ + hash_text, task_title_is_generic, IdentitySource, RuntimeInfo, SessionTitleSource, TaskSpan, + TaskSpanId, TASK_SPAN_SCHEMA_VERSION, +}; + +/// Gives the event a session and a project. Sessions with neither a project +/// nor a title are not kept, so tests of other behavior need one. +fn stamp_session(event: &mut UsageEvent, raw_id: &str) { + let hash = hash_text(raw_id); + event.session.session_id = format!("session_{}", &hash[..24]); + event.session.local_session_id_hash = Some(hash); + if event.project.is_none() { + event.project = Some(session_test_project()); + } +} + +fn session_test_project() -> ProjectInfo { + ProjectInfo { + project_id: "project-sessions".to_string(), + project_label: None, + repo_remote_hash: None, + repo_label: None, + branch_hash: None, + branch_label: None, + path_hash: Some("path-sessions".to_string()), + path_label: None, + } +} + +fn message_runtime(user: u64, assistant: u64) -> RuntimeInfo { + RuntimeInfo { + runtime_name: None, + host_id: None, + latency_ms: None, + latency_source: None, + time_to_first_token_ms: None, + prompt_eval_duration_ms: None, + eval_duration_ms: None, + total_messages: Some(user + assistant), + user_messages: Some(user), + assistant_messages: Some(assistant), + developer_messages: None, + } +} + +#[test] +fn session_rollup_sums_mixed_events() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-mixed"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 2, 9, 0, 0) + .single() + .expect("start"); + let later = start + chrono::Duration::minutes(12); + + let mut first = test_store_event(&source, start, "first"); + stamp_session(&mut first, "raw-session"); + first.session.ended_at = None; + first.usage.requests = Some(2); + first.usage.input_tokens = Some(10); + first.usage.output_tokens = Some(4); + first.usage.total_tokens = Some(14); + first.model = Some(ModelInfo { + normalized_name: Some("gpt-5".to_string()), + ..ModelInfo::default() + }); + first.runtime = Some(message_runtime(1, 1)); + first.cost.estimated_api_equivalent_micro_usd = Some(20_000); + first.created_at = start; + + let mut second = test_store_event(&source, later, "second"); + stamp_session(&mut second, "raw-session"); + second.session.started_at = start; + second.session.ended_at = Some(later); + second.usage.requests = None; + second.usage.input_tokens = Some(5); + second.usage.output_tokens = Some(7); + second.usage.total_tokens = Some(12); + second.model = Some(ModelInfo { + normalized_name: Some("gpt-5-mini".to_string()), + ..ModelInfo::default() + }); + second.usage.input_tokens = Some(30); + second.usage.output_tokens = Some(1); + second.usage.total_tokens = Some(31); + second.runtime = Some(message_runtime(2, 1)); + second.cost.estimated_api_equivalent_micro_usd = Some(5_000); + second.created_at = later; + + assert!(store.insert_event(&first).expect("insert first")); + assert!(store.insert_event(&second).expect("insert second")); + + let rollups = store + .session_rollups_in_period( + None, + later + chrono::Duration::days(1), + &SessionFilter::default(), + SessionSort::Started, + 10, + 0, + ) + .expect("rollups"); + assert_eq!(rollups.len(), 1); + let rollup = &rollups[0]; + assert_eq!(rollup.schema_version, "session_rollup.v1"); + assert_eq!(rollup.session_id, first.session.session_id); + assert_eq!(rollup.usage.input_tokens, Some(40)); + assert_eq!(rollup.usage.output_tokens, Some(5)); + assert_eq!(rollup.usage.computed_total(), 45); + assert_eq!(rollup.requests, 3); + assert_eq!(rollup.cost.estimated_micro_usd(), Some(25_000)); + assert_eq!(rollup.user_messages, Some(3)); + assert_eq!(rollup.assistant_messages, Some(2)); + assert_eq!(rollup.total_messages, Some(5)); + assert_eq!(rollup.primary_model.as_deref(), Some("gpt-5-mini")); + assert_eq!(rollup.started_at, start); + assert_eq!(rollup.ended_at, later); + assert_eq!(rollup.duration_seconds, Some(12 * 60)); + assert_eq!(rollup.models.len(), 2); +} + +#[test] +fn session_rollup_orders_inverted_event_times() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-inverted"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 2, 9, 0, 0) + .single() + .expect("start"); + let later = start + chrono::Duration::minutes(12); + let mut event = test_store_event(&source, later, "inverted"); + stamp_session(&mut event, "raw-inverted"); + event.session.started_at = later; + event.session.ended_at = Some(start); + event.created_at = start; + assert!(store.insert_event(&event).expect("insert")); + + let rollups = store + .session_rollups_in_period( + None, + later + chrono::Duration::days(1), + &SessionFilter::default(), + SessionSort::Started, + 10, + 0, + ) + .expect("rollups"); + assert_eq!(rollups.len(), 1); + assert_eq!(rollups[0].started_at, later); + assert_eq!(rollups[0].ended_at, later); + assert_eq!(rollups[0].duration_seconds, Some(0)); +} + +#[test] +fn session_rollup_refreshes_on_insert_and_delete() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-refresh"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 3, 9, 0, 0) + .single() + .expect("start"); + let mut kept = test_store_event(&source, start, "kept"); + stamp_session(&mut kept, "refresh-session"); + kept.usage.total_tokens = Some(10); + kept.parse_evidence = Some(ParseEvidence { + event_key_version: "test".to_string(), + source_file_path_hash: Some("file-kept".to_string()), + source_line_number: None, + source_record_id: None, + model_inferred: false, + timestamp_inferred: false, + account_identity_source: IdentitySource::Unresolved, + }); + let mut dropped = test_store_event(&source, start + chrono::Duration::minutes(1), "dropped"); + stamp_session(&mut dropped, "refresh-session"); + dropped.session.started_at = start; + dropped.usage.total_tokens = Some(7); + dropped.parse_evidence = Some(ParseEvidence { + event_key_version: "test".to_string(), + source_file_path_hash: Some("file-dropped".to_string()), + source_line_number: None, + source_record_id: None, + model_inferred: false, + timestamp_inferred: false, + account_identity_source: IdentitySource::Unresolved, + }); + store.insert_event(&kept).expect("kept"); + store.insert_event(&dropped).expect("dropped"); + let before = store.dirty_session_rollups().expect("dirty"); + assert_eq!(before.len(), 1); + assert_eq!(before[0].usage.computed_total(), 17); + + let impact = store + .delete_events_for_source_file_hashes(&source.source_id, &["file-dropped".to_string()]) + .expect("delete"); + assert_eq!(impact.deleted, 1); + let after = store.dirty_session_rollups().expect("after"); + assert_eq!(after.len(), 1); + assert_eq!(after[0].usage.computed_total(), 10); + + store + .delete_events_for_sources(std::slice::from_ref(&source.source_id)) + .expect("delete source"); + assert!(store.dirty_session_rollups().expect("gone").is_empty()); +} + +#[test] +fn session_titles_prefer_event_then_task_span_then_archive() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-titles"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 4, 9, 0, 0) + .single() + .expect("start"); + let raw_id = "provider-session-9"; + let mut event = test_store_event(&source, start, "titled"); + stamp_session(&mut event, raw_id); + store.insert_event(&event).expect("insert"); + + let generic = "hello"; + assert!(task_title_is_generic(Some(generic))); + store + .upsert_task_spans(&[task_span(&source, raw_id, generic, start)]) + .expect("generic span"); + let unlabeled = store.dirty_session_rollups().expect("unlabeled"); + assert_eq!(unlabeled[0].title, None); + + let title = "Repair the session indexer"; + assert!(!task_title_is_generic(Some(title))); + store + .upsert_task_spans(&[task_span(&source, raw_id, title, start)]) + .expect("span"); + let from_span = store.dirty_session_rollups().expect("span title"); + assert_eq!(from_span[0].title.as_deref(), Some(title)); + assert_eq!( + from_span[0].title_source, + Some(SessionTitleSource::TaskSpan) + ); + + event.session.title = Some("Event title wins".to_string()); + store.insert_event(&event).expect("retitle"); + let from_event = store.dirty_session_rollups().expect("event title"); + assert_eq!(from_event[0].title.as_deref(), Some("Event title wins")); + assert_eq!(from_event[0].title_source, Some(SessionTitleSource::Event)); +} + +#[test] +fn session_queries_filter_period_and_sort() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "claude_code", + "test", + "0", + Path::new("/tmp/session-query"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let early = Utc + .with_ymd_and_hms(2026, 6, 1, 8, 0, 0) + .single() + .expect("early"); + let late = Utc + .with_ymd_and_hms(2026, 6, 10, 8, 0, 0) + .single() + .expect("late"); + let mut small = test_store_event(&source, early, "small"); + stamp_session(&mut small, "small-session"); + small.provider = "claude_code".to_string(); + small.usage.total_tokens = Some(5); + small.session.ended_at = Some(early + chrono::Duration::seconds(30)); + let mut large = test_store_event(&source, late, "large"); + stamp_session(&mut large, "large-session"); + large.provider = "claude_code".to_string(); + large.usage.total_tokens = Some(50); + large.session.ended_at = Some(late + chrono::Duration::seconds(90)); + store.insert_event(&small).expect("small"); + store.insert_event(&large).expect("large"); + + let filter = SessionFilter { + provider: Some("claude_code".to_string()), + ..SessionFilter::default() + }; + let page = store + .session_rollups_in_period( + Some(Utc.with_ymd_and_hms(2026, 6, 5, 0, 0, 0).unwrap()), + late + chrono::Duration::days(1), + &filter, + SessionSort::Tokens, + 10, + 0, + ) + .expect("page"); + assert_eq!(page.len(), 1); + assert_eq!(page[0].usage.computed_total(), 50); + + let sorted = store + .session_rollups_in_period(None, late, &filter, SessionSort::Tokens, 10, 0) + .expect("sorted"); + assert_eq!(sorted.len(), 2); + assert_eq!(sorted[0].usage.computed_total(), 50); + assert_eq!(sorted[1].usage.computed_total(), 5); + let stats = store + .session_stats_in_period(None, late, &filter) + .expect("stats"); + assert_eq!(stats.sessions, 2); + assert_eq!(stats.total_tokens, 55); + assert_eq!(stats.avg_tokens, Some(27)); + assert_eq!(stats.median_tokens, Some(27)); + assert_eq!(stats.median_duration_seconds, Some(60)); +} + +fn task_span( + source: &statsai_core::SourceLocation, + raw_session_id: &str, + title: &str, + started_at: chrono::DateTime, +) -> TaskSpan { + TaskSpan { + schema_version: TASK_SPAN_SCHEMA_VERSION.to_string(), + span_id: TaskSpanId(format!("span-{raw_session_id}-{title}")), + provider: source.provider.clone(), + source_id: source.source_id.clone(), + span_kind: "test".to_string(), + source_record_id: None, + source_file_path_hash: None, + summary_id: None, + session_id: Some(raw_session_id.to_string()), + thread_id: None, + title: title.to_string(), + normalized_title: statsai_core::normalize_task_title(title), + title_source: Some("test".to_string()), + summary_preview: None, + todo_excerpt: None, + issue_keys: Vec::new(), + branch_family: None, + project_bucket: "bucket".to_string(), + project: None, + git: None, + usage: UsageCounts::default(), + estimated_cost_usd: None, + estimated_cost_micro_usd: None, + event_count: 0, + has_usage_evidence: false, + total_messages: 0, + user_messages: 0, + assistant_messages: 0, + developer_messages: 0, + linked_event_ids: Vec::new(), + confidence: Confidence::Medium, + is_meta: task_title_is_generic(Some(title)), + started_at, + ended_at: Some(started_at), + duration_seconds: Some(0), + } +} + +fn record_evidence(record_id: &str) -> ParseEvidence { + ParseEvidence { + event_key_version: "provider_record_usage_event.v1".to_string(), + source_file_path_hash: None, + source_line_number: None, + source_record_id: Some(record_id.to_string()), + model_inferred: false, + timestamp_inferred: false, + account_identity_source: IdentitySource::Unresolved, + } +} + +#[test] +fn record_keyed_events_do_not_form_sessions() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "cursor", + "test", + "0", + Path::new("/tmp/session-cursor"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 5, 9, 0, 0) + .single() + .expect("start"); + + // A local Cursor row is keyed by its own record, so its session hash is + // the record hash. + let mut row = test_store_event(&source, start, "cursor-row"); + stamp_session(&mut row, "cursor-row-identity"); + let row_hash = hash_text("cursor-row-identity"); + row.parse_evidence = Some(record_evidence(&format!( + "provider_record_usage_event.v1:cursor_usage_event:{row_hash}" + ))); + store.insert_event(&row).expect("row"); + + // A cloud agent groups its rows under one agent id. + let mut agent = test_store_event(&source, start, "cursor-agent-row"); + stamp_session(&mut agent, "cursor_agent:agent-1"); + let record_hash = hash_text("agent-row-identity"); + agent.parse_evidence = Some(record_evidence(&format!( + "provider_record_usage_event.v1:cursor_usage_event:{record_hash}" + ))); + store.insert_event(&agent).expect("agent"); + + let sessions = store.dirty_session_rollups().expect("sessions"); + assert_eq!(sessions.len(), 1); + assert_eq!(sessions[0].session_id, agent.session.session_id); + + assert_eq!(store.rebuild_session_rollups().expect("rebuild"), 1); + let rebuilt = store.dirty_session_rollups().expect("rebuilt"); + assert_eq!(rebuilt.len(), 1); + assert_eq!(rebuilt[0].session_id, agent.session.session_id); +} + +#[test] +fn single_instant_sessions_have_unknown_duration() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-instant"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 6, 9, 0, 0) + .single() + .expect("start"); + let mut lone = test_store_event(&source, start, "lone"); + stamp_session(&mut lone, "lone-session"); + lone.created_at = start; + store.insert_event(&lone).expect("lone"); + + let mut spanned = test_store_event(&source, start, "spanned-first"); + stamp_session(&mut spanned, "spanned-session"); + spanned.created_at = start; + store.insert_event(&spanned).expect("first"); + let mut later = test_store_event( + &source, + start + chrono::Duration::minutes(3), + "spanned-last", + ); + stamp_session(&mut later, "spanned-session"); + later.created_at = start + chrono::Duration::minutes(3); + store.insert_event(&later).expect("last"); + + let sessions = store.dirty_session_rollups().expect("sessions"); + let duration = |session_id: &str| { + sessions + .iter() + .find(|session| session.session_id == session_id) + .expect("session") + .duration_seconds + }; + assert_eq!(duration(&lone.session.session_id), None); + assert_eq!(duration(&spanned.session.session_id), Some(180)); + let stats = store + .session_stats_in_period( + None, + start + chrono::Duration::days(1), + &SessionFilter::default(), + ) + .expect("stats"); + assert_eq!(stats.median_duration_seconds, Some(180)); +} + +#[test] +fn codex_rollout_stem_span_titles_join_uuid_sessions() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-codex-stem"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 7, 9, 0, 0) + .single() + .expect("start"); + let uuid = "019c537d-7d4b-7c33-bfc9-785a18e58f8c"; + let mut event = test_store_event(&source, start, "codex-stem"); + stamp_session(&mut event, uuid); + store.insert_event(&event).expect("insert"); + + let title = "Repair the rollout title join"; + store + .upsert_task_spans(&[task_span( + &source, + &format!("2026/02/12/rollout-2026-02-12T23-14-18-{uuid}"), + title, + start, + )]) + .expect("span"); + store.rebuild_session_rollups().expect("rebuild"); + let sessions = store.dirty_session_rollups().expect("sessions"); + assert_eq!(sessions[0].title.as_deref(), Some(title)); +} + +#[test] +fn rebuild_resends_unchanged_sessions_without_restamping() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-rebuild"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 8, 9, 0, 0) + .single() + .expect("start"); + let mut event = test_store_event(&source, start, "rebuild"); + stamp_session(&mut event, "rebuild-session"); + store.insert_event(&event).expect("insert"); + let ids = store + .dirty_session_rollups() + .expect("dirty") + .into_iter() + .map(|session| session.session_id) + .collect::>(); + let before = store.dirty_session_rollups().expect("before"); + store.mark_session_rollups_synced(&ids).expect("synced"); + + store.rebuild_session_rollups().expect("rebuild"); + let resent = store.dirty_session_rollups().expect("resent"); + assert_eq!(resent, before); +} + +#[test] +fn provider_named_sessions_keep_plain_names_but_drop_placeholders() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-provider-titles"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 9, 9, 0, 0) + .single() + .expect("start"); + let plain = "Review uncommitted changes"; + assert!(task_title_is_generic(Some(plain))); + let mut review = test_store_event(&source, start, "review"); + stamp_session(&mut review, "review-session"); + review.session.title = Some(plain.to_string()); + store.insert_event(&review).expect("review"); + + let mut placeholder = test_store_event(&source, start, "placeholder"); + stamp_session(&mut placeholder, "placeholder-session"); + placeholder.session.title = Some("New session - 2026-04-30T16:41:41.413Z".to_string()); + store.insert_event(&placeholder).expect("placeholder"); + + let mut bare = test_store_event(&source, start, "bare-placeholder"); + stamp_session(&mut bare, "bare-placeholder-session"); + bare.session.title = Some("New session".to_string()); + store.insert_event(&bare).expect("bare placeholder"); + + let sessions = store.dirty_session_rollups().expect("sessions"); + let title = |event: &UsageEvent| { + sessions + .iter() + .find(|session| session.session_id == event.session.session_id) + .expect("session") + .title + .clone() + }; + assert_eq!(title(&review).as_deref(), Some(plain)); + assert_eq!(title(&placeholder), None); + assert_eq!(title(&bare), None); +} + +#[test] +fn session_backfill_resends_every_session_to_each_target() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-backfill-resend"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 10, 9, 0, 0) + .single() + .expect("start"); + let mut event = test_store_event(&source, start, "backfill"); + stamp_session(&mut event, "backfill-session"); + store.insert_event(&event).expect("insert"); + let sessions = store.all_session_rollups().expect("sessions"); + store + .record_session_rollups_synced("http", "https://api.example.test", &sessions) + .expect("record"); + assert!(store + .pending_session_rollups_for_sync("http", "https://api.example.test", &sessions) + .expect("pending") + .is_empty()); + + store + .conn + .execute( + "DELETE FROM local_metadata WHERE key LIKE 'session_rollups_backfilled_%'", + [], + ) + .expect("forget backfill"); + store + .backfill_session_rollups_if_needed() + .expect("backfill"); + + let pending = store + .pending_session_rollups_for_sync("http", "https://api.example.test", &sessions) + .expect("pending after backfill"); + assert_eq!(pending, sessions); + + // The acknowledgement survives, so a session gone from the next snapshot + // is still retired on the target. + let snapshot = statsai_core::SyncAuthoritativeSnapshot { + session_rollup_ids: Some(Vec::new()), + ..statsai_core::SyncAuthoritativeSnapshot::default() + }; + assert!(store + .sync_target_has_retired_entities("http", "https://api.example.test", &snapshot) + .expect("retired")); +} + +#[test] +fn sessions_without_project_or_title_are_not_kept() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "cursor", + "test", + "0", + Path::new("/tmp/session-unidentifiable"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 11, 9, 0, 0) + .single() + .expect("start"); + let mut agent = test_store_event(&source, start, "agent"); + stamp_session(&mut agent, "cursor_agent:agent-2"); + agent.project = None; + store.insert_event(&agent).expect("agent"); + assert!(store.dirty_session_rollups().expect("sessions").is_empty()); + + agent.session.title = Some("Upgrade the importer".to_string()); + store.insert_event(&agent).expect("titled agent"); + assert_eq!(store.dirty_session_rollups().expect("titled").len(), 1); + + assert_eq!(store.rebuild_session_rollups().expect("rebuild"), 1); +} + +#[test] +fn active_time_counts_turns_and_skips_idle_gaps() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "claude_code", + "test", + "0", + Path::new("/tmp/session-active"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let at = |hour: u32, minute: u32| { + Utc.with_ymd_and_hms(2026, 6, 12, hour, minute, 0) + .single() + .expect("time") + }; + // Two turns of a Claude-style session, the second resumed hours later: + // each event carries the prompt that started its turn. + for (record, prompt, message) in [ + ("first-a", at(10, 0), at(10, 1)), + ("first-b", at(10, 0), at(10, 5)), + ("second", at(14, 0), at(14, 2)), + ] { + let mut event = test_store_event(&source, message, record); + stamp_session(&mut event, "active-session"); + event.created_at = message; + event.session.turn_started_at = Some(prompt); + store.insert_event(&event).expect("insert"); + } + + let sessions = store.dirty_session_rollups().expect("sessions"); + assert_eq!(sessions.len(), 1); + assert_eq!(sessions[0].started_at, at(10, 0)); + // Wall-clock span keeps the idle afternoon; active time does not. + assert_eq!(sessions[0].duration_seconds, Some(4 * 3600 + 2 * 60)); + assert_eq!(sessions[0].active_seconds, Some(7 * 60)); +} + +#[test] +fn active_time_merges_overlapping_turn_intervals() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-active-overlap"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let at = |minute: u32| { + Utc.with_ymd_and_hms(2026, 6, 13, 9, minute, 0) + .single() + .expect("time") + }; + // Codex-style turns carry their own start and completion. A record + // without an interval adds nothing. + for (record, start, end) in [ + ("turn-a", at(0), Some(at(10))), + ("turn-b", at(5), Some(at(12))), + ("turn-c", at(30), Some(at(31))), + ("record", at(40), None), + ] { + let mut event = test_store_event(&source, start, record); + stamp_session(&mut event, "overlap-session"); + event.created_at = end.unwrap_or(start); + event.session.ended_at = end; + store.insert_event(&event).expect("insert"); + } + + let sessions = store.dirty_session_rollups().expect("sessions"); + assert_eq!(sessions[0].active_seconds, Some(13 * 60)); + assert_eq!(sessions[0].duration_seconds, Some(40 * 60)); +} + +#[test] +fn session_aggregate_events_do_not_count_as_active_time() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "opencode", + "test", + "0", + Path::new("/tmp/session-aggregate"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 14, 9, 0, 0) + .single() + .expect("start"); + let end = start + chrono::Duration::days(3); + let mut aggregate = test_store_event(&source, end, "aggregate"); + stamp_session(&mut aggregate, "aggregate-session"); + aggregate.source.source_type = "sqlite:session".to_string(); + aggregate.session.started_at = start; + aggregate.session.ended_at = Some(end); + aggregate.created_at = end; + store.insert_event(&aggregate).expect("insert"); + + let sessions = store.dirty_session_rollups().expect("sessions"); + assert_eq!(sessions[0].duration_seconds, Some(3 * 86_400)); + assert_eq!(sessions[0].active_seconds, None); +} + +#[test] +fn reported_turn_duration_bounds_a_late_completion() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-late-completion"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 7, 23, 17, 16, 0) + .single() + .expect("start"); + // Codex reported 21 minutes of work but wrote the completion when the + // thread resumed four days later. + let completed = start + chrono::Duration::days(4); + let mut turn = test_store_event(&source, start, "late-turn"); + stamp_session(&mut turn, "late-session"); + turn.session.ended_at = Some(completed); + turn.session.duration_seconds = Some(21 * 60); + turn.created_at = completed; + store.insert_event(&turn).expect("insert"); + + let sessions = store.dirty_session_rollups().expect("sessions"); + assert_eq!(sessions[0].active_seconds, Some(21 * 60)); + assert_eq!(sessions[0].duration_seconds, Some(4 * 86_400)); +} + +#[test] +fn session_stats_handle_no_matching_sessions() { + let store = Store::in_memory().expect("store"); + let stats = store + .session_stats_in_period(None, Utc::now(), &SessionFilter::default()) + .expect("stats"); + assert_eq!(stats.sessions, 0); + assert_eq!(stats.avg_tokens, None); + assert_eq!(stats.top_model, None); +} + +#[test] +fn top_model_sums_each_sessions_per_model_usage() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-top-model"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 15, 9, 0, 0) + .single() + .expect("start"); + let model = |name: &str| ModelInfo { + name: Some(name.to_string()), + normalized_name: Some(name.to_string()), + provider_model_id: None, + speed: None, + reasoning_level: None, + reasoning_level_raw: None, + }; + // One session is model-a 60 and model-b 40; another is model-b 50. By + // primary model alone, model-a would win 100 to 50. + for (record, session, name, tokens) in [ + ("mixed-a", "mixed-session", "model-a", 60), + ("mixed-b", "mixed-session", "model-b", 40), + ("single-b", "single-session", "model-b", 50), + ] { + let mut event = test_store_event(&source, start, record); + stamp_session(&mut event, session); + event.model = Some(model(name)); + event.usage = UsageCounts { + input_tokens: Some(tokens), + total_tokens: Some(tokens), + ..UsageCounts::default() + }; + store.insert_event(&event).expect("insert"); + } + + let stats = store + .session_stats_in_period( + None, + start + chrono::Duration::days(1), + &SessionFilter::default(), + ) + .expect("stats"); + assert_eq!(stats.top_model.as_deref(), Some("model-b")); +} + +#[test] +fn a_later_event_without_a_project_keeps_the_sessions_project() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-project-gap"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 16, 9, 0, 0) + .single() + .expect("start"); + let mut first = test_store_event(&source, start, "with-project"); + stamp_session(&mut first, "project-gap-session"); + store.insert_event(&first).expect("first"); + let mut latest = test_store_event(&source, start + chrono::Duration::minutes(5), "no-project"); + stamp_session(&mut latest, "project-gap-session"); + latest.project = None; + store.insert_event(&latest).expect("latest"); + + let sessions = store.dirty_session_rollups().expect("sessions"); + assert_eq!(sessions.len(), 1); + assert_eq!(sessions[0].project, Some(session_test_project())); +} + +#[test] +fn deleting_a_source_rebuilds_sessions_it_shared() { + let store = Store::in_memory().expect("store"); + let older = SourceLocation::local_adapter( + "claude_code", + "test", + "0", + Path::new("/tmp/session-shared-older"), + LocationOrigin::Configured, + ); + let newer = SourceLocation::local_adapter( + "claude_code", + "test", + "0", + Path::new("/tmp/session-shared-newer"), + LocationOrigin::Configured, + ); + store.upsert_source(&older).expect("older source"); + store.upsert_source(&newer).expect("newer source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 17, 9, 0, 0) + .single() + .expect("start"); + for (source, record, at, tokens) in [ + (&older, "older-event", start, 10), + ( + &newer, + "newer-event", + start + chrono::Duration::minutes(5), + 7, + ), + ] { + let mut event = test_store_event(source, at, record); + stamp_session(&mut event, "shared-session"); + event.usage.total_tokens = Some(tokens); + store.insert_event(&event).expect("insert"); + } + let total = |store: &Store| { + store + .all_session_rollups() + .expect("sessions") + .iter() + .map(|session| session.usage.computed_total()) + .collect::>() + }; + assert_eq!(total(&store), vec![17]); + + // The older source's tokens leave the session, which the newer one keeps. + store + .delete_events_for_sources(std::slice::from_ref(&older.source_id)) + .expect("delete older"); + assert_eq!(total(&store), vec![7]); + + // And the other way round: the surviving events keep the session. + let mut again = test_store_event(&older, start, "older-again"); + stamp_session(&mut again, "shared-session"); + again.usage.total_tokens = Some(10); + store.insert_event(&again).expect("reinsert older"); + store + .delete_events_for_sources(std::slice::from_ref(&newer.source_id)) + .expect("delete newer"); + assert_eq!(total(&store), vec![10]); +} + +#[test] +fn deleting_the_span_that_titled_a_session_clears_the_title() { + let store = Store::in_memory().expect("store"); + let usage_source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-span-usage"), + LocationOrigin::Configured, + ); + let span_source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-span-titles"), + LocationOrigin::Configured, + ); + store.upsert_source(&usage_source).expect("usage source"); + store.upsert_source(&span_source).expect("span source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 18, 9, 0, 0) + .single() + .expect("start"); + let raw_id = "span-titled-session"; + let mut event = test_store_event(&usage_source, start, "span-usage"); + stamp_session(&mut event, raw_id); + store.insert_event(&event).expect("insert"); + let title = "Repair the span title cleanup"; + store + .upsert_task_spans(&[task_span(&span_source, raw_id, title, start)]) + .expect("span"); + assert_eq!( + store.all_session_rollups().expect("titled")[0] + .title + .as_deref(), + Some(title) + ); + + let targets = store + .task_span_targets_for_sources(std::slice::from_ref(&span_source.source_id)) + .expect("targets"); + store + .delete_task_span_targets_in_tx(&targets) + .expect("delete spans"); + let sessions = store.all_session_rollups().expect("after"); + assert_eq!(sessions.len(), 1); + assert_eq!(sessions[0].title, None); +} + +#[test] +fn a_provider_name_renames_a_stored_session_without_its_events() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "claude_code", + "test", + "0", + Path::new("/tmp/session-names"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 19, 9, 0, 0) + .single() + .expect("start"); + let mut event = test_store_event(&source, start, "named"); + stamp_session(&mut event, "named-session"); + event.session.title = Some("Title when parsed".to_string()); + store.insert_event(&event).expect("insert"); + + let rename = |title: &str| statsai_core::SessionName { + local_session_id_hash: hash_text("named-session"), + title: title.to_string(), + parent_local_session_id_hash: None, + }; + assert_eq!( + store + .upsert_session_names(&[rename("Renamed later")]) + .expect("rename"), + 1 + ); + let sessions = store.all_session_rollups().expect("sessions"); + assert_eq!(sessions[0].title.as_deref(), Some("Renamed later")); + assert_eq!(sessions[0].title_source, Some(SessionTitleSource::Event)); + + // An unchanged name writes nothing. + assert_eq!( + store + .upsert_session_names(&[rename("Renamed later")]) + .expect("same"), + 0 + ); +} + +#[test] +fn a_sub_agent_borrows_its_parents_current_name() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-parent-names"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 19, 9, 0, 0) + .single() + .expect("start"); + let mut event = test_store_event(&source, start, "child"); + stamp_session(&mut event, "child-thread"); + store.insert_event(&event).expect("insert"); + + let name = |raw: &str, title: &str, parent: Option<&str>| statsai_core::SessionName { + local_session_id_hash: hash_text(raw), + title: title.to_string(), + parent_local_session_id_hash: parent.map(hash_text), + }; + let title = || { + store.all_session_rollups().expect("sessions")[0] + .title + .clone() + }; + store + .upsert_session_names(&[ + name("parent-thread", "Design tool/plugin", None), + name("child-thread", "Popper", Some("parent-thread")), + ]) + .expect("names"); + assert_eq!(title().as_deref(), Some("Design tool/plugin · Popper")); + + // Renaming the parent renames the sub-agent, whose rollout is not re-read. + store + .upsert_session_names(&[name("parent-thread", "Plugin API", None)]) + .expect("rename parent"); + assert_eq!(title().as_deref(), Some("Plugin API · Popper")); + + // A name of its own wins, and a borrowed one does not replace it. + store + .upsert_session_names(&[name("child-thread", "Review plugin", None)]) + .expect("own name"); + assert_eq!( + store + .upsert_session_names(&[name("child-thread", "Popper", Some("parent-thread"))]) + .expect("borrowed again"), + 0 + ); + assert_eq!(title().as_deref(), Some("Review plugin")); +} diff --git a/crates/statsai-store/src/tests/scan_state.rs b/crates/statsai-store/src/tests/scan_state.rs index 2a8c8ab..179e3fe 100644 --- a/crates/statsai-store/src/tests/scan_state.rs +++ b/crates/statsai-store/src/tests/scan_state.rs @@ -60,6 +60,7 @@ fn scan_file_replacement_rolls_back_deletions_when_cache_update_fails() { reconciled_file_hashes: &[file_hash], events: &[], summaries: &[], + session_names: &[], activity_invocations: &[], activity_coverage: &[], activity_persist_mode: ActivityPersistMode::ReplaceFiles, @@ -325,6 +326,7 @@ fn scan_file_replacement_clears_links_to_events_it_retired() { reconciled_file_hashes: std::slice::from_ref(&file_hash), events: std::slice::from_ref(&surviving), summaries: &[], + session_names: &[], activity_invocations: &[], activity_coverage: &[], activity_persist_mode: ActivityPersistMode::ReplaceFiles, diff --git a/crates/statsai-store/src/tests/support.rs b/crates/statsai-store/src/tests/support.rs index a3f1e66..0f6fc58 100644 --- a/crates/statsai-store/src/tests/support.rs +++ b/crates/statsai-store/src/tests/support.rs @@ -46,6 +46,7 @@ pub(super) fn test_store_event( started_at: now, ended_at: None, duration_seconds: None, + turn_started_at: None, }, model: None, usage: UsageCounts { diff --git a/crates/statsai-store/src/tests/sync_state/mod.rs b/crates/statsai-store/src/tests/sync_state/mod.rs index 073b309..ae1a399 100644 --- a/crates/statsai-store/src/tests/sync_state/mod.rs +++ b/crates/statsai-store/src/tests/sync_state/mod.rs @@ -146,6 +146,7 @@ fn record_rollup_chunk_sync_success_retries_busy_database() { account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: summaries.clone(), task_buckets: Vec::new(), @@ -400,6 +401,7 @@ fn sync_preferences_round_trip_and_normalize_tasks() { include_projects: false, include_tasks: true, include_activity: false, + include_sessions: false, }) .expect("save sync preferences"); @@ -409,6 +411,25 @@ fn sync_preferences_round_trip_and_normalize_tasks() { include_projects: true, include_tasks: true, include_activity: false, + include_sessions: false, + } + ); + + store + .set_sync_preferences(SyncPreferences { + include_projects: false, + include_tasks: false, + include_activity: false, + include_sessions: true, + }) + .expect("save session sync preferences"); + assert_eq!( + store.sync_preferences().expect("session sync preferences"), + SyncPreferences { + include_projects: true, + include_tasks: false, + include_activity: false, + include_sessions: true, } ); } diff --git a/crates/statsai-sync/src/lib.rs b/crates/statsai-sync/src/lib.rs index d844606..6fd5a92 100644 --- a/crates/statsai-sync/src/lib.rs +++ b/crates/statsai-sync/src/lib.rs @@ -336,6 +336,12 @@ fn validate_sync_ack(batch: &SyncBatch, ack: &SyncAck) -> Result<()> { ack.accepted.account_evidence_summaries, ack.duplicates.account_evidence_summaries, )?; + validate_sync_ack_counts( + "sessions", + batch.sessions.len() as u64, + ack.accepted.sessions, + ack.duplicates.sessions, + )?; Ok(()) } @@ -459,6 +465,7 @@ mod tests { account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: Vec::new(), @@ -824,6 +831,7 @@ mod tests { started_at: Utc::now(), ended_at: None, duration_seconds: None, + turn_started_at: None, }, model: None, usage: statsai_core::UsageCounts::default(), diff --git a/crates/statsai/src/cli/args/mod.rs b/crates/statsai/src/cli/args/mod.rs index 027d8e6..0d80179 100644 --- a/crates/statsai/src/cli/args/mod.rs +++ b/crates/statsai/src/cli/args/mod.rs @@ -1,4 +1,4 @@ -use clap::{Args, Parser, Subcommand}; +use clap::{Args, Parser, Subcommand, ValueEnum}; use statsai::snapshot; use std::path::PathBuf; use std::str::FromStr; @@ -8,6 +8,7 @@ mod activity; mod quota; mod report; mod scan; +mod sessions; mod source; mod subscription; mod sync; @@ -18,6 +19,7 @@ pub(crate) use activity::*; pub(crate) use quota::*; pub(crate) use report::*; pub(crate) use scan::*; +pub(crate) use sessions::*; pub(crate) use source::*; pub(crate) use subscription::*; pub(crate) use sync::*; @@ -46,6 +48,8 @@ pub(crate) enum Command { Scan(ScanCommand), #[command(about = "Show usage reports (weekly, monthly, all-time, or a date range)")] Report(ReportCommand), + #[command(about = "List local session rollups")] + Sessions(SessionsCommand), #[command(about = "Manage configured source paths")] Source(SourceCommand), #[command(about = "List canonical provider accounts")] diff --git a/crates/statsai/src/cli/args/sessions.rs b/crates/statsai/src/cli/args/sessions.rs new file mode 100644 index 0000000..03a5fb9 --- /dev/null +++ b/crates/statsai/src/cli/args/sessions.rs @@ -0,0 +1,43 @@ +use super::*; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +pub(crate) enum SessionSortArg { + Started, + Tokens, + Cost, + Duration, + Messages, +} + +impl SessionSortArg { + pub(crate) fn store_sort(self) -> statsai_store::SessionSort { + match self { + Self::Started => statsai_store::SessionSort::Started, + Self::Tokens => statsai_store::SessionSort::Tokens, + Self::Cost => statsai_store::SessionSort::Cost, + Self::Duration => statsai_store::SessionSort::Duration, + Self::Messages => statsai_store::SessionSort::Messages, + } + } +} + +#[derive(Debug, Args)] +pub(crate) struct SessionsCommand { + #[arg( + long, + help = "Range start (YYYY-MM-DD or RFC3339). Defaults to the last 7 days" + )] + pub(crate) from: Option, + #[arg(long, help = "Range end (YYYY-MM-DD or RFC3339). Defaults to now")] + pub(crate) to: Option, + #[arg(long, help = "Provider filter")] + pub(crate) provider: Option, + #[arg(long, help = "Project id, key, or label")] + pub(crate) project: Option, + #[arg(long, value_enum, default_value = "started", help = "Sort descending")] + pub(crate) sort: SessionSortArg, + #[arg(long, default_value_t = 50, help = "Maximum sessions to print")] + pub(crate) limit: usize, + #[arg(long, help = "Output as JSON")] + pub(crate) json: bool, +} diff --git a/crates/statsai/src/cli/args/sync.rs b/crates/statsai/src/cli/args/sync.rs index 24de0ce..18a5b79 100644 --- a/crates/statsai/src/cli/args/sync.rs +++ b/crates/statsai/src/cli/args/sync.rs @@ -55,8 +55,8 @@ pub(crate) struct SyncCommand { pub(crate) include_projects: bool, #[arg( long, - conflicts_with_all = ["include_projects", "include_tasks"], - help = "Disable project metadata sync for this device and future syncs" + conflicts_with_all = ["include_projects", "include_tasks", "include_sessions"], + help = "Disable project metadata sync for this device and future syncs; hosted sessions are retired" )] pub(crate) exclude_projects: bool, #[arg( @@ -83,4 +83,16 @@ pub(crate) struct SyncCommand { help = "Disable hosted agent activity sync and prune names on the next successful HTTP sync" )] pub(crate) exclude_activity: bool, + #[arg( + long, + conflicts_with_all = ["exclude_sessions", "exclude_projects"], + help = "Enable hosted session sync for this device and future syncs (implies --include-projects)" + )] + pub(crate) include_sessions: bool, + #[arg( + long, + conflicts_with = "include_sessions", + help = "Disable hosted session sync for this device and future syncs" + )] + pub(crate) exclude_sessions: bool, } diff --git a/crates/statsai/src/cli/dispatch.rs b/crates/statsai/src/cli/dispatch.rs index 90ae129..a8ddb6a 100644 --- a/crates/statsai/src/cli/dispatch.rs +++ b/crates/statsai/src/cli/dispatch.rs @@ -26,6 +26,7 @@ pub(crate) fn run() -> Result<()> { match command { Command::Scan(command) => scan(command, &store, &device_id), Command::Report(command) => report(command, &store), + Command::Sessions(command) => sessions(command, &store), Command::Source(command) => source(command, &store, &device_id), Command::Account(command) => account(command, &store), Command::Subscription(command) => subscription(command, &store), @@ -59,6 +60,7 @@ pub(crate) fn command_reprices_persisted_usage(command: &Command) -> bool { command, Command::Scan(_) | Command::Report(_) + | Command::Sessions(_) | Command::Import(_) | Command::Export(_) | Command::Task(_) diff --git a/crates/statsai/src/cli/mod.rs b/crates/statsai/src/cli/mod.rs index ab38a50..e3e365a 100644 --- a/crates/statsai/src/cli/mod.rs +++ b/crates/statsai/src/cli/mod.rs @@ -12,6 +12,7 @@ pub(super) mod report; pub(super) mod scan; pub(super) mod schema; pub(super) mod service; +pub(super) mod sessions; pub(super) mod source; pub(super) mod status; pub(super) mod store_admin; @@ -34,6 +35,7 @@ pub(crate) use report::*; pub(crate) use scan::*; pub(crate) use schema::*; pub(crate) use service::*; +pub(crate) use sessions::*; pub(crate) use source::*; pub(crate) use status::*; pub(crate) use store_admin::*; diff --git a/crates/statsai/src/cli/scan/mod.rs b/crates/statsai/src/cli/scan/mod.rs index f4f8a22..303f8d2 100644 --- a/crates/statsai/src/cli/scan/mod.rs +++ b/crates/statsai/src/cli/scan/mod.rs @@ -466,6 +466,7 @@ pub(crate) fn scan_with_adapters( } let insert_started_at = Instant::now(); let insert_result = store.insert_events_with_resolution(&scan.events)?; + store.upsert_session_names(&scan.session_names)?; inserted_count += insert_result.inserted; insert_events_duration_ms += insert_started_at.elapsed().as_millis() as u64; rewrite_quota_usage_event_ids( diff --git a/crates/statsai/src/cli/sessions.rs b/crates/statsai/src/cli/sessions.rs new file mode 100644 index 0000000..5974011 --- /dev/null +++ b/crates/statsai/src/cli/sessions.rs @@ -0,0 +1,157 @@ +use anyhow::Result; +use chrono::{DateTime, Utc}; +use serde_json::json; +use statsai_core::{ + micro_usd_to_cents_rounded, report_period_from_range, ReportPeriod, SessionRollupV1, +}; +use statsai_store::{SessionFilter, SessionStats, Store}; + +use super::args::SessionsCommand; +use super::format::{format_cost, format_u64, truncate_label}; +use super::source::canonical_provider; + +pub(crate) struct SessionsView { + pub(crate) label: String, + pub(crate) stats: SessionStats, + pub(crate) sessions: Vec, +} + +pub(crate) fn sessions(command: SessionsCommand, store: &Store) -> Result<()> { + let view = sessions_view(&command, store, Utc::now())?; + if command.json { + println!("{}", serde_json::to_string_pretty(&sessions_json(&view))?); + } else { + print_sessions_table(&view); + } + Ok(()) +} + +pub(crate) fn sessions_view( + command: &SessionsCommand, + store: &Store, + now: DateTime, +) -> Result { + let period = match (&command.from, &command.to) { + (None, None) => ReportPeriod::LastDays(7), + (from, to) => report_period_from_range(from.as_deref(), to.as_deref(), now)?, + }; + let (since, until) = period.published_window(now); + let provider = command + .provider + .as_deref() + .map(canonical_provider) + .transpose()?; + let filter = SessionFilter { + provider, + project_key: command.project.clone(), + account: None, + }; + let stats = store.session_stats_in_period(since, until, &filter)?; + let sessions = store.session_rollups_in_period( + since, + until, + &filter, + command.sort.store_sort(), + command.limit, + 0, + )?; + Ok(SessionsView { + label: period.label(now), + stats, + sessions, + }) +} + +fn sessions_json(view: &SessionsView) -> serde_json::Value { + json!({ + "label": view.label, + "stats": view.stats, + "sessions": view.sessions, + }) +} + +fn print_sessions_table(view: &SessionsView) { + let stats = &view.stats; + println!("statsai sessions: {}", view.label); + println!("sessions: {}", format_u64(stats.sessions)); + println!("tokens: {}", format_u64(stats.total_tokens)); + println!( + "avg tokens/session: {}", + stats + .avg_tokens + .map(format_u64) + .unwrap_or_else(|| "unknown".to_string()) + ); + println!( + "median active time: {}", + format_duration(stats.median_active_seconds) + ); + println!("messages: {}", format_u64(stats.total_messages)); + println!( + "est. cost: {}", + format_cost(Some(micro_usd_to_cents_rounded(stats.total_cost_micro_usd))) + ); + println!( + "top model: {}", + stats.top_model.as_deref().unwrap_or("unknown") + ); + println!(); + println!( + "{:<16} {:<12} {:<18} {:<24} {:>6} {:>10} {:>10} {:>10} {:>10} {:>10} {:>10}", + "started", + "provider", + "model", + "title", + "msgs", + "input", + "output", + "total", + "est_cost", + "active", + "span" + ); + for session in &view.sessions { + println!( + "{:<16} {:<12} {:<18} {:<24} {:>6} {:>10} {:>10} {:>10} {:>10} {:>10} {:>10}", + session.started_at.format("%Y-%m-%d %H:%M"), + truncate_label(&session.provider, 12), + truncate_label(session.primary_model.as_deref().unwrap_or("unknown"), 18), + truncate_label(session.title.as_deref().unwrap_or("—"), 24), + session + .total_messages + .map(format_u64) + .unwrap_or_else(|| "—".to_string()), + format_u64(session.usage.input_tokens.unwrap_or(0)), + format_u64(session.usage.output_tokens.unwrap_or(0)), + format_u64(session.usage.computed_total()), + format_cost( + session + .cost + .estimated_micro_usd() + .map(micro_usd_to_cents_rounded) + ), + format_duration(session.active_seconds), + format_duration(session.duration_seconds) + ); + } +} + +pub(crate) fn format_duration(seconds: Option) -> String { + let Some(seconds) = seconds else { + return "unknown".to_string(); + }; + let days = seconds / 86_400; + let hours = seconds / 3600; + let minutes = (seconds % 3600) / 60; + let secs = seconds % 60; + // Past two days, hours stop being readable: "20d 18h", not "498h 15m". + if hours >= 48 { + format!("{days}d {:02}h", hours % 24) + } else if hours > 0 { + format!("{hours}h {minutes:02}m") + } else if minutes > 0 { + format!("{minutes}m {secs:02}s") + } else { + format!("{secs}s") + } +} diff --git a/crates/statsai/src/cli/sync/batch.rs b/crates/statsai/src/cli/sync/batch.rs index d3686d9..69367d2 100644 --- a/crates/statsai/src/cli/sync/batch.rs +++ b/crates/statsai/src/cli/sync/batch.rs @@ -9,9 +9,10 @@ use statsai_core::{ }; use statsai_core::{ project_contains_file_paths, sanitize_code_change_metric_for_sync, - sanitize_task_bucket_for_sync, IdentitySource, ProjectInfo, ProviderAccount, - SourceAccountAssignment, SourceKind, SourceLocation, Subscription, SyncAuthoritativeSnapshot, - SyncBatch, TaskVerificationCursor, UsageEvent, UsageSummary, SYNC_BATCH_SCHEMA_VERSION, + sanitize_session_rollup_for_sync, sanitize_task_bucket_for_sync, IdentitySource, ProjectInfo, + ProviderAccount, SourceAccountAssignment, SourceKind, SourceLocation, Subscription, + SyncAuthoritativeSnapshot, SyncBatch, TaskVerificationCursor, UsageEvent, UsageSummary, + SYNC_BATCH_SCHEMA_VERSION, }; use statsai_store::{QuotaQuery, Store}; #[cfg(test)] @@ -277,6 +278,28 @@ pub(crate) fn build_sync_batch_with_identity_key( } else { all_activity_coverage.clone() }; + let all_sessions = if sync_preferences.include_sessions { + store + .all_session_rollups()? + .into_iter() + .map(|rollup| { + let mut rollup = sanitize_session_rollup_for_sync(rollup); + if !include_projects { + rollup.project = None; + } + rollup + }) + .collect::>() + } else { + Vec::new() + }; + let sessions = if !sync_preferences.include_sessions { + Vec::new() + } else if payload_mode == SyncPayloadMode::Rollups && !(command.full || state.is_none()) { + store.pending_session_rollups_for_sync(&command.sink, target, &all_sessions)? + } else { + all_sessions.clone() + }; if payload_mode == SyncPayloadMode::Rollups { let label = rollup_mode_label(command); @@ -327,6 +350,18 @@ pub(crate) fn build_sync_batch_with_identity_key( .iter() .map(|coverage| coverage.coverage_id.clone()) .collect(), + session_rollup_ids: store.authoritative_session_rollup_ids( + &command.sink, + target, + sync_preferences.include_sessions, + include_projects, + || { + Ok(all_sessions + .iter() + .map(|rollup| rollup.session_id.clone()) + .collect()) + }, + )?, }; let failed_without_resume = state.as_ref().is_some_and(|state| { state.failure_count > 0 && state.pending_resume_batch_id.is_none() @@ -403,6 +438,7 @@ pub(crate) fn build_sync_batch_with_identity_key( quota_cycle_contributions, activity_rollups, activity_coverage, + sessions, authoritative_snapshot, created_at, }, @@ -466,6 +502,13 @@ pub(crate) fn record_rollup_sync_chunk_success( .map(|coverage| coverage.coverage_id.clone()) .collect::>(), )?; + store.mark_session_rollups_synced( + &batch + .sessions + .iter() + .map(|rollup| rollup.session_id.clone()) + .collect::>(), + )?; snapshot::invalidate_dashboard_cache(); Ok(()) } @@ -516,6 +559,7 @@ pub(crate) fn record_sync_batch_success( // the next sync to that target resent every rollup. store.record_activity_rollups_synced(sink, target, &batch.activity_rollups)?; store.record_activity_coverage_synced(sink, target, &batch.activity_coverage)?; + store.record_session_rollups_synced(sink, target, &batch.sessions)?; store.mark_activity_rollups_synced( &batch .activity_rollups @@ -530,6 +574,13 @@ pub(crate) fn record_sync_batch_success( .map(|coverage| coverage.coverage_id.clone()) .collect::>(), )?; + store.mark_session_rollups_synced( + &batch + .sessions + .iter() + .map(|rollup| rollup.session_id.clone()) + .collect::>(), + )?; snapshot::invalidate_dashboard_cache(); Ok(()) } diff --git a/crates/statsai/src/cli/sync/chunking/budget.rs b/crates/statsai/src/cli/sync/chunking/budget.rs index dcfadf9..821185a 100644 --- a/crates/statsai/src/cli/sync/chunking/budget.rs +++ b/crates/statsai/src/cli/sync/chunking/budget.rs @@ -102,6 +102,32 @@ pub(crate) fn estimate_http_rollup_d1_queries(batch: &SyncBatch) -> usize { // ownership write, matching quota-cycle batching. let activity_queries = usize::from(!batch.activity_rollups.is_empty()) * 2 + usize::from(!batch.activity_coverage.is_empty()) * 2; + // Sessions are one freshness lookup, one upsert, and one ownership write. + // A closing snapshot that names session_rollup_ids also deletes orphans. + let session_queries = usize::from(!batch.sessions.is_empty()) * 3 + + usize::from( + batch + .authoritative_snapshot + .as_ref() + .is_some_and(|snapshot| { + snapshot.part_index.saturating_add(1) == snapshot.part_count + && snapshot.session_rollup_ids.is_some() + }), + ); + // Any session or activity row advances the dashboard revision. Summaries + // already pay for the full snapshot refresh above. + let dashboard_snapshot_timestamp_queries = usize::from( + !batch.sessions.is_empty() + || !batch.activity_rollups.is_empty() + || !batch.activity_coverage.is_empty(), + ); + // Plan and evidence account ids already reserve the account-plan alias + // lookup. Sessions and activity share that one query, so charge it only + // when they name an account and nothing else already did. + let session_activity_alias_queries = usize::from( + !http_rollup_account_plan_names_provider_account(batch) + && http_rollup_session_or_activity_names_provider_account(batch), + ); let code_change_owner_metadata_refresh_queries = usize::from( batch.schema_version == SYNC_BATCH_SCHEMA_VERSION && batch @@ -127,6 +153,9 @@ pub(crate) fn estimate_http_rollup_d1_queries(batch: &SyncBatch) -> usize { + code_change_metric_queries + quota_cycle_contribution_queries + activity_queries + + session_queries + + dashboard_snapshot_timestamp_queries + + session_activity_alias_queries + code_change_owner_metadata_refresh_queries + estimate_http_rollup_task_queries(batch) + final_sync_bookkeeping_queries @@ -373,12 +402,21 @@ pub(crate) fn http_rollup_project_count(batch: &SyncBatch) -> usize { .summaries .iter() .filter_map(http_rollup_summary_project_key) + .chain( + batch + .sessions + .iter() + .filter_map(|session| session.project.as_ref().and_then(http_rollup_project_key)), + ) .collect::>() .len() } pub(crate) fn http_rollup_summary_project_key(summary: &UsageSummary) -> Option { - let project = summary.project.as_ref()?; + summary.project.as_ref().and_then(http_rollup_project_key) +} + +pub(crate) fn http_rollup_project_key(project: &statsai_core::ProjectInfo) -> Option { if !project_has_stable_identity(project) { return None; } @@ -396,12 +434,26 @@ pub(crate) fn http_rollup_project_location_count(batch: &SyncBatch) -> usize { .summaries .iter() .filter_map(http_rollup_summary_project_location_key) + .chain(batch.sessions.iter().filter_map(|session| { + session + .project + .as_ref() + .and_then(http_rollup_project_location_key) + })) .collect::>() .len() } pub(crate) fn http_rollup_summary_project_location_key(summary: &UsageSummary) -> Option { - let project = summary.project.as_ref()?; + summary + .project + .as_ref() + .and_then(http_rollup_project_location_key) +} + +pub(crate) fn http_rollup_project_location_key( + project: &statsai_core::ProjectInfo, +) -> Option { if !project_has_stable_identity(project) { return None; } @@ -413,3 +465,33 @@ pub(crate) fn http_rollup_summary_project_location_key(summary: &UsageSummary) - } Some(format!("project:{}", project.project_id)) } + +fn http_rollup_account_plan_names_provider_account(batch: &SyncBatch) -> bool { + !unique_non_empty_provider_account_ids( + batch + .account_plan_observations + .iter() + .map(|observation| observation.provider_account_id.0.as_str()) + .chain( + batch + .account_evidence_summaries + .iter() + .map(|summary| summary.provider_account_id.0.as_str()), + ), + ) + .is_empty() +} + +fn http_rollup_session_or_activity_names_provider_account(batch: &SyncBatch) -> bool { + batch.sessions.iter().any(|session| { + session + .provider_account_id + .as_ref() + .is_some_and(|account_id| !account_id.0.trim().is_empty()) + }) || batch.activity_rollups.iter().any(|rollup| { + rollup + .provider_account_id + .as_ref() + .is_some_and(|account_id| !account_id.0.trim().is_empty()) + }) +} diff --git a/crates/statsai/src/cli/sync/chunking/mod.rs b/crates/statsai/src/cli/sync/chunking/mod.rs index 98a3d01..57a1610 100644 --- a/crates/statsai/src/cli/sync/chunking/mod.rs +++ b/crates/statsai/src/cli/sync/chunking/mod.rs @@ -31,6 +31,7 @@ pub(crate) enum HttpRollupIndexedChunkKind { QuotaCycles, ActivityRollups, ActivityCoverage, + Sessions, Snapshot, } @@ -48,6 +49,7 @@ impl HttpRollupIndexedChunkKind { Self::QuotaCycles, Self::ActivityRollups, Self::ActivityCoverage, + Self::Sessions, Self::Snapshot, ]; @@ -74,6 +76,7 @@ impl HttpRollupIndexedChunkKind { Self::QuotaCycles => "quota_cycles", Self::ActivityRollups => "activity_rollups", Self::ActivityCoverage => "activity_coverage", + Self::Sessions => "sessions", Self::Snapshot => "snapshot", } } @@ -97,6 +100,8 @@ const HTTP_ROLLUP_QUOTA_CYCLE_CONTRIBUTIONS_PER_BATCH: usize = 100; pub(crate) const HTTP_ROLLUP_ACTIVITY_ROWS_PER_BATCH: usize = 500; +pub(crate) const HTTP_ROLLUP_SESSIONS_PER_BATCH: usize = 200; + pub(crate) const HTTP_ROLLUP_D1_QUERY_BUDGET: usize = 45; const HTTP_ROLLUP_D1_QUERY_CHUNK_SIZE: usize = 90; @@ -156,7 +161,9 @@ fn split_authoritative_snapshot( quota_cycle_contribution_ids: Vec::new(), activity_rollup_ids: Vec::new(), activity_coverage_ids: Vec::new(), + session_rollup_ids: None, }; + let session_rollup_ids = snapshot.session_rollup_ids; let mut parts = Vec::new(); let mut current = empty_part(); @@ -194,7 +201,27 @@ fn split_authoritative_snapshot( ); append_ids!(snapshot.activity_rollup_ids, activity_rollup_ids); append_ids!(snapshot.activity_coverage_ids, activity_coverage_ids); - if authoritative_snapshot_id_count(¤t) > 0 || parts.is_empty() { + if let Some(ids) = session_rollup_ids { + if ids.is_empty() { + // An empty array is the retirement signal. It adds no IDs, so it + // can ride on the current part even when that part is already full. + current.session_rollup_ids = Some(Vec::new()); + } else { + for id in ids { + if authoritative_snapshot_id_count(¤t) == max_ids { + parts.push(std::mem::replace(&mut current, empty_part())); + } + current + .session_rollup_ids + .get_or_insert_with(Vec::new) + .push(id); + } + } + } + if authoritative_snapshot_id_count(¤t) > 0 + || current.session_rollup_ids.is_some() + || parts.is_empty() + { parts.push(current); } let part_count = u32::try_from(parts.len()).expect("snapshot part count fits u32"); @@ -217,6 +244,7 @@ fn authoritative_snapshot_id_count(snapshot: &SyncAuthoritativeSnapshot) -> usiz + snapshot.quota_cycle_contribution_ids.len() + snapshot.activity_rollup_ids.len() + snapshot.activity_coverage_ids.len() + + snapshot.session_rollup_ids.as_ref().map_or(0, Vec::len) } pub(crate) fn split_http_rollup_sync_batches_without_snapshot(batch: &SyncBatch) -> Vec { @@ -232,13 +260,15 @@ pub(crate) fn split_http_rollup_sync_batches_without_snapshot(batch: &SyncBatch) || !batch.code_change_metrics.is_empty() || !batch.quota_cycle_contributions.is_empty() || !batch.activity_rollups.is_empty() - || !batch.activity_coverage.is_empty(); + || !batch.activity_coverage.is_empty() + || !batch.sessions.is_empty(); if !has_task_payload && batch.summaries.len() <= HTTP_ROLLUP_SUMMARIES_PER_BATCH && batch.code_change_metrics.len() <= HTTP_ROLLUP_CODE_CHANGE_METRICS_PER_BATCH && batch.quota_cycle_contributions.len() <= HTTP_ROLLUP_QUOTA_CYCLE_CONTRIBUTIONS_PER_BATCH && batch.activity_rollups.len() <= HTTP_ROLLUP_ACTIVITY_ROWS_PER_BATCH && batch.activity_coverage.len() <= HTTP_ROLLUP_ACTIVITY_ROWS_PER_BATCH + && sessions_fit_one_http_chunk(batch) && metadata_count <= HTTP_ROLLUP_METADATA_RECORDS_PER_BATCH { return fit_http_rollup_batches_to_d1_budget(vec![batch.clone()]); @@ -268,12 +298,17 @@ pub(crate) fn split_http_rollup_sync_batches_without_snapshot(batch: &SyncBatch) .activity_coverage .len() .div_ceil(HTTP_ROLLUP_ACTIVITY_ROWS_PER_BATCH); + let session_chunks = batch + .sessions + .len() + .div_ceil(HTTP_ROLLUP_SESSIONS_PER_BATCH); let mut chunks = Vec::with_capacity( total_chunks + metadata_chunks + code_change_chunks + quota_cycle_chunks + activity_chunks + + session_chunks + task_chunks.len(), ); @@ -294,6 +329,7 @@ pub(crate) fn split_http_rollup_sync_batches_without_snapshot(batch: &SyncBatch) batch, HTTP_ROLLUP_ACTIVITY_ROWS_PER_BATCH, )); + chunks.extend(split_http_session_chunks(batch)); chunks.extend(split_http_rollup_summary_chunks( batch, HTTP_ROLLUP_SUMMARIES_PER_BATCH, @@ -319,6 +355,14 @@ fn empty_http_rollup_chunk(batch: &SyncBatch, suffix: &str) -> SyncBatch { chunk.quota_cycle_contributions.clear(); chunk.activity_rollups.clear(); chunk.activity_coverage.clear(); + chunk.sessions.clear(); chunk.authoritative_snapshot = None; chunk } + +fn sessions_fit_one_http_chunk(batch: &SyncBatch) -> bool { + batch.sessions.len() <= HTTP_ROLLUP_SESSIONS_PER_BATCH + && serde_json::to_string(&batch.sessions) + .map(|payload| payload.len() <= TASK_SYNC_SQL_MAX_JSON_BYTES_PER_CHUNK) + .unwrap_or(false) +} diff --git a/crates/statsai/src/cli/sync/chunking/payloads.rs b/crates/statsai/src/cli/sync/chunking/payloads.rs index b2d7930..0fe95a9 100644 --- a/crates/statsai/src/cli/sync/chunking/payloads.rs +++ b/crates/statsai/src/cli/sync/chunking/payloads.rs @@ -1,6 +1,18 @@ use super::*; pub(crate) fn split_http_rollup_sync_batch_after_budget_error(batch: &SyncBatch) -> Vec { + if !batch.sessions.is_empty() && has_non_session_payload(batch) { + let mut without_sessions = batch.clone(); + without_sessions.sessions.clear(); + let mut chunks = split_http_session_chunks(batch); + chunks.extend(split_http_rollup_sync_batch_after_budget_error( + &without_sessions, + )); + return chunks; + } + if batch.sessions.len() > 1 { + return split_http_session_chunks_halved(batch); + } if (!batch.activity_rollups.is_empty() || !batch.activity_coverage.is_empty()) && has_non_activity_payload(batch) { @@ -143,6 +155,7 @@ pub(crate) fn has_non_code_change_payload(batch: &SyncBatch) -> bool { || !batch.quota_cycle_contributions.is_empty() || !batch.activity_rollups.is_empty() || !batch.activity_coverage.is_empty() + || !batch.sessions.is_empty() } pub(crate) fn has_non_quota_cycle_payload(batch: &SyncBatch) -> bool { @@ -154,6 +167,19 @@ pub(crate) fn has_non_quota_cycle_payload(batch: &SyncBatch) -> bool { || !batch.code_change_metrics.is_empty() || !batch.activity_rollups.is_empty() || !batch.activity_coverage.is_empty() + || !batch.sessions.is_empty() +} + +pub(crate) fn has_non_session_payload(batch: &SyncBatch) -> bool { + http_rollup_metadata_count(batch) > 0 + || !batch.events.is_empty() + || !batch.summaries.is_empty() + || !batch.task_buckets.is_empty() + || !batch.task_verifications.is_empty() + || !batch.code_change_metrics.is_empty() + || !batch.quota_cycle_contributions.is_empty() + || !batch.activity_rollups.is_empty() + || !batch.activity_coverage.is_empty() } pub(crate) fn has_non_activity_payload(batch: &SyncBatch) -> bool { @@ -164,6 +190,7 @@ pub(crate) fn has_non_activity_payload(batch: &SyncBatch) -> bool { || !batch.task_verifications.is_empty() || !batch.code_change_metrics.is_empty() || !batch.quota_cycle_contributions.is_empty() + || !batch.sessions.is_empty() } /// Records the backend writes one statement per row for. @@ -260,6 +287,62 @@ pub(crate) fn split_http_quota_cycle_contribution_chunks( .collect() } +pub(crate) fn split_http_session_chunks(batch: &SyncBatch) -> Vec { + let mut chunks = Vec::new(); + let mut current = Vec::new(); + let mut current_bytes = 2usize; + for session in &batch.sessions { + let row_bytes = serde_json::to_string(session) + .map(|row| row.len()) + .unwrap_or(0); + let next_bytes = if current.is_empty() { + 2 + row_bytes + } else { + current_bytes + 1 + row_bytes + }; + if !current.is_empty() + && (current.len() >= HTTP_ROLLUP_SESSIONS_PER_BATCH + || next_bytes > TASK_SYNC_SQL_MAX_JSON_BYTES_PER_CHUNK) + { + chunks.push(session_chunk(batch, chunks.len(), ¤t)); + current.clear(); + current_bytes = 2; + } + current.push(session.clone()); + current_bytes = if current.len() == 1 { + 2 + row_bytes + } else { + current_bytes + 1 + row_bytes + }; + } + if !current.is_empty() { + chunks.push(session_chunk(batch, chunks.len(), ¤t)); + } + chunks +} + +fn split_http_session_chunks_halved(batch: &SyncBatch) -> Vec { + let midpoint = batch.sessions.len().div_ceil(2).max(1); + let mut chunks = Vec::new(); + for (index, sessions) in batch.sessions.chunks(midpoint).enumerate() { + chunks.push(session_chunk(batch, index, sessions)); + } + chunks +} + +fn session_chunk( + batch: &SyncBatch, + index: usize, + sessions: &[statsai_core::SessionRollupV1], +) -> SyncBatch { + let mut chunk = empty_http_rollup_chunk( + batch, + &HttpRollupIndexedChunkKind::Sessions.suffix(index + 1), + ); + chunk.sessions = sessions.to_vec(); + chunk +} + pub(crate) fn split_http_activity_chunks(batch: &SyncBatch, chunk_size: usize) -> Vec { let chunk_size = chunk_size.max(1); let mut chunks = Vec::new(); @@ -376,6 +459,7 @@ pub(crate) fn split_http_rollup_single_metadata_kind( | HttpRollupIndexedChunkKind::QuotaCycles | HttpRollupIndexedChunkKind::ActivityRollups | HttpRollupIndexedChunkKind::ActivityCoverage + | HttpRollupIndexedChunkKind::Sessions | HttpRollupIndexedChunkKind::Snapshot => Vec::new(), } } @@ -411,6 +495,7 @@ pub(crate) fn split_http_rollup_summary_chunks( chunk.quota_cycle_contributions.clear(); chunk.activity_rollups.clear(); chunk.activity_coverage.clear(); + chunk.sessions.clear(); chunk.authoritative_snapshot = None; chunk }) diff --git a/crates/statsai/src/cli/sync/http.rs b/crates/statsai/src/cli/sync/http.rs index 5bfb220..a8dbccb 100644 --- a/crates/statsai/src/cli/sync/http.rs +++ b/crates/statsai/src/cli/sync/http.rs @@ -12,7 +12,8 @@ use std::time::Duration as StdDuration; use super::super::args::SyncCommand; use super::batch::{record_rollup_sync_chunk_success, record_sync_batch_success}; use super::chunking::{ - has_non_code_change_payload, has_non_quota_cycle_payload, http_rollup_metadata_count, + has_non_activity_payload, has_non_code_change_payload, has_non_quota_cycle_payload, + has_non_session_payload, http_rollup_metadata_count, split_http_rollup_sync_batch_after_budget_error, split_http_rollup_sync_batches, HttpRollupIndexedChunkKind, HTTP_ROLLUP_SUMMARIES_PER_BATCH, }; @@ -379,6 +380,9 @@ pub(crate) fn should_retry_http_rollup_chunk_after_error( { return false; } + // Relinks rewrite three tables per location. The server decides that after + // reading stored projects, so a 413 on a splittable session or activity + // chunk has to be retried smaller even when the local estimate fit. chunk.summaries.len() > 1 || chunk.sources.len() > 1 || chunk.accounts.len() > 1 @@ -390,10 +394,16 @@ pub(crate) fn should_retry_http_rollup_chunk_after_error( || chunk.task_verifications.len() > 1 || chunk.code_change_metrics.len() > 1 || chunk.quota_cycle_contributions.len() > 1 + || chunk.activity_rollups.len() > 1 + || chunk.activity_coverage.len() > 1 + || chunk.sessions.len() > 1 || (!chunk.task_buckets.is_empty() && !chunk.task_verifications.is_empty()) || (http_rollup_metadata_count(chunk) > 0 && !chunk.summaries.is_empty()) || (!chunk.code_change_metrics.is_empty() && has_non_code_change_payload(chunk)) || (!chunk.quota_cycle_contributions.is_empty() && has_non_quota_cycle_payload(chunk)) + || (!chunk.sessions.is_empty() && has_non_session_payload(chunk)) + || ((!chunk.activity_rollups.is_empty() || !chunk.activity_coverage.is_empty()) + && has_non_activity_payload(chunk)) } fn http_rollup_retry_error_label(error: &anyhow::Error) -> &'static str { diff --git a/crates/statsai/src/cli/sync/mod.rs b/crates/statsai/src/cli/sync/mod.rs index 9d5bba4..d65e291 100644 --- a/crates/statsai/src/cli/sync/mod.rs +++ b/crates/statsai/src/cli/sync/mod.rs @@ -36,6 +36,7 @@ pub(crate) fn effective_sync_preferences( if command.exclude_projects { preferences.include_projects = false; preferences.include_tasks = false; + preferences.include_sessions = false; } if command.include_tasks { preferences.include_projects = true; @@ -50,6 +51,13 @@ pub(crate) fn effective_sync_preferences( if command.exclude_activity { preferences.include_activity = false; } + if command.include_sessions { + preferences.include_projects = true; + preferences.include_sessions = true; + } + if command.exclude_sessions { + preferences.include_sessions = false; + } Ok(preferences.normalized()) } @@ -63,7 +71,7 @@ fn apply_sync_preference_overrides( if preferences != original { store.set_sync_preferences(preferences)?; eprintln!( - "sync preferences updated: projects={} tasks={} activity={}", + "sync preferences updated: projects={} tasks={} activity={} sessions={}", if preferences.include_projects { "enabled" } else { @@ -78,11 +86,17 @@ fn apply_sync_preference_overrides( "enabled" } else { "disabled" + }, + if preferences.include_sessions { + "enabled" + } else { + "disabled" } ); if (!original.include_projects && preferences.include_projects) || (!original.include_tasks && preferences.include_tasks) || (!original.include_activity && preferences.include_activity) + || (!original.include_sessions && preferences.include_sessions) { eprintln!( "sync preferences changed privacy/backfill scope; the next sync may resend historical summaries to update the hosted mirror" @@ -101,6 +115,20 @@ fn apply_sync_preference_overrides( "activity sync disabled: the next successful HTTP sync sends an empty activity snapshot so hosted names are pruned" ); } + if !original.include_sessions && preferences.include_sessions { + eprintln!( + "session sync enabled: the next sync backfills session rollups, and later snapshots retire hosted sessions this device no longer has" + ); + } + if original.include_projects && !preferences.include_projects { + eprintln!( + "project sync disabled: the next successful HTTP sync retires this device's hosted sessions, which carry project names and paths" + ); + } else if original.include_sessions && !preferences.include_sessions { + eprintln!( + "session sync disabled: hosted sessions stay in place until session sync is enabled again" + ); + } } Ok(preferences) } @@ -174,20 +202,21 @@ pub(crate) fn sync(command: SyncCommand, store: &Store, device_id: &str) -> Resu if command.dry_run { eprintln!( - "dry run: sink={} mode={} include_projects={} include_tasks={} include_activity={} sources={} events={} summaries={} task_buckets={} task_verifications={} activity_rollups={} activity_coverage={}", + "dry run: sink={} mode={} include_projects={} include_tasks={} include_activity={} include_sessions={} sources={} events={} summaries={} task_buckets={} task_verifications={} activity_rollups={} activity_coverage={} sessions={}", command.sink, sync_payload_mode_name(payload_mode), sync_preferences.include_projects, sync_preferences.include_tasks, sync_preferences.include_activity, + sync_preferences.include_sessions, batch.sources.len(), batch.events.len(), - batch.summaries.len() - , + batch.summaries.len(), batch.task_buckets.len(), batch.task_verifications.len(), batch.activity_rollups.len(), batch.activity_coverage.len(), + batch.sessions.len(), ); return Ok(()); } @@ -903,6 +932,8 @@ mod tests { exclude_tasks: false, include_activity: false, exclude_activity: false, + include_sessions: false, + exclude_sessions: false, } } diff --git a/crates/statsai/src/cli/tests/mod.rs b/crates/statsai/src/cli/tests/mod.rs index 159246c..bb1a76f 100644 --- a/crates/statsai/src/cli/tests/mod.rs +++ b/crates/statsai/src/cli/tests/mod.rs @@ -8,6 +8,7 @@ mod import; mod quota; mod report; mod scan; +mod sessions; mod source; mod status; mod subscription; diff --git a/crates/statsai/src/cli/tests/scan/files.rs b/crates/statsai/src/cli/tests/scan/files.rs index 0ab5310..f07be94 100644 --- a/crates/statsai/src/cli/tests/scan/files.rs +++ b/crates/statsai/src/cli/tests/scan/files.rs @@ -772,3 +772,114 @@ fn scan_rewrites_task_span_links_to_canonical_event_ids() { vec![existing_event.event_id.clone()] ); } + +/// The real Codex adapter, pointed at one source instead of the machine's. +struct PinnedCodexAdapter { + source: SourceLocation, +} + +impl ProviderAdapter for PinnedCodexAdapter { + fn id(&self) -> &'static str { + statsai_adapters::CodexAdapter.id() + } + + fn version(&self) -> &'static str { + statsai_adapters::CodexAdapter.version() + } + + fn provider(&self) -> &'static str { + statsai_adapters::CodexAdapter.provider() + } + + fn discover(&self) -> Vec { + vec![self.source.clone()] + } + + fn scan_candidates(&self, source: &SourceLocation) -> Result> { + statsai_adapters::CodexAdapter.scan_candidates(source) + } + + fn scan( + &self, + source: &SourceLocation, + options: &ScanOptions, + ) -> Result { + statsai_adapters::CodexAdapter.scan(source, options) + } +} + +#[test] +fn cached_scan_picks_up_a_codex_thread_rename() { + let dir = tempfile::tempdir().expect("tempdir"); + let codex_root = dir.path().join("codex"); + let sessions = codex_root.join("sessions"); + std::fs::create_dir_all(&sessions).expect("sessions"); + let index_path = codex_root.join("session_index.jsonl"); + std::fs::write( + &index_path, + "{\"id\":\"thread-1\",\"thread_name\":\"First name\"}\n", + ) + .expect("session index"); + std::fs::write( + sessions.join("rollout.jsonl"), + [ + r#"{"timestamp":"2026-06-01T08:00:00Z","type":"session_meta","payload":{"id":"thread-1"}}"#, + r#"{"timestamp":"2026-06-01T08:00:00Z","type":"turn_context","payload":{"model":"gpt-5"}}"#, + r#"{"timestamp":"2026-06-01T08:00:03Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":60,"cached_input_tokens":20,"output_tokens":30,"reasoning_output_tokens":10,"total_tokens":120},"total_token_usage":{"input_tokens":60,"cached_input_tokens":20,"output_tokens":30,"reasoning_output_tokens":10,"total_tokens":120}}}}"#, + ] + .join("\n") + + "\n", + ) + .expect("rollout"); + + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "codex-local-jsonl", + statsai_adapters::CodexAdapter.version(), + &codex_root, + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let scan = || { + scan_with_adapters( + ScanCommand { + provider: None, + include_tasks: false, + preview: false, + no_cache: false, + replace: false, + verbose: false, + explain: false, + }, + &store, + "device-test", + vec![Box::new(PinnedCodexAdapter { + source: source.clone(), + })], + ) + .expect("scan"); + }; + let titles = || { + store + .all_session_rollups() + .expect("sessions") + .into_iter() + .map(|session| session.title) + .collect::>() + }; + + scan(); + assert_eq!(titles(), vec![Some("First name".to_string())]); + + // Codex appends the rename to the index and leaves the rollout alone. + std::thread::sleep(std::time::Duration::from_millis(5)); + std::fs::write( + &index_path, + "{\"id\":\"thread-1\",\"thread_name\":\"First name\"}\n\ + {\"id\":\"thread-1\",\"thread_name\":\"Renamed thread\"}\n", + ) + .expect("renamed index"); + scan(); + assert_eq!(titles(), vec![Some("Renamed thread".to_string())]); +} diff --git a/crates/statsai/src/cli/tests/sessions.rs b/crates/statsai/src/cli/tests/sessions.rs new file mode 100644 index 0000000..5e4ad9e --- /dev/null +++ b/crates/statsai/src/cli/tests/sessions.rs @@ -0,0 +1,110 @@ +use super::support::*; +use super::*; + +#[test] +fn sessions_cli_parses_filters() { + let cli = Cli::try_parse_from([ + "statsai", + "sessions", + "--from", + "2026-06-01", + "--to", + "2026-06-15", + "--provider", + "codex", + "--project", + "statsai", + "--sort", + "tokens", + "--limit", + "10", + "--json", + ]) + .expect("parse"); + match cli.command { + Command::Sessions(command) => { + assert_eq!(command.from.as_deref(), Some("2026-06-01")); + assert_eq!(command.to.as_deref(), Some("2026-06-15")); + assert_eq!(command.provider.as_deref(), Some("codex")); + assert_eq!(command.project.as_deref(), Some("statsai")); + assert_eq!(command.sort, SessionSortArg::Tokens); + assert_eq!(command.limit, 10); + assert!(command.json); + } + other => panic!("unexpected command {other:?}"), + } +} + +#[test] +fn sessions_view_reads_rollups_for_the_default_week() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/sessions-cli"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let now = Utc + .with_ymd_and_hms(2026, 6, 20, 12, 0, 0) + .single() + .expect("now"); + let mut recent = test_event( + "codex", + &source, + now - Duration::days(1), + None, + TokenParts::total(40), + ); + recent.session.session_id = "session_recent".to_string(); + recent.session.title = Some("Recent indexing work".to_string()); + recent.model = Some(ModelInfo { + normalized_name: Some("gpt-5".to_string()), + ..ModelInfo::default() + }); + let mut old = test_event( + "codex", + &source, + now - Duration::days(30), + None, + TokenParts::total(9), + ); + old.session.session_id = "session_old".to_string(); + store.insert_event(&recent).expect("recent"); + store.insert_event(&old).expect("old"); + + let command = SessionsCommand { + from: None, + to: None, + provider: Some("codex".to_string()), + project: None, + sort: SessionSortArg::Started, + limit: 50, + json: true, + }; + let view = sessions_view(&command, &store, now).expect("view"); + assert_eq!(view.label, "last 7 days"); + assert_eq!(view.stats.sessions, 1); + assert_eq!(view.stats.total_tokens, 40); + assert_eq!(view.stats.top_model.as_deref(), Some("gpt-5")); + assert_eq!(view.sessions.len(), 1); + assert_eq!( + view.sessions[0].title.as_deref(), + Some("Recent indexing work") + ); + assert_eq!( + view.sessions[0].title_source, + Some(statsai_core::SessionTitleSource::Event) + ); +} + +#[test] +fn session_durations_switch_to_days_past_two_days() { + use crate::cli::sessions::format_duration; + assert_eq!(format_duration(None), "unknown"); + assert_eq!(format_duration(Some(45)), "45s"); + assert_eq!(format_duration(Some(3 * 60 + 5)), "3m 05s"); + assert_eq!(format_duration(Some(47 * 3600 + 59 * 60)), "47h 59m"); + assert_eq!(format_duration(Some(498 * 3600 + 15 * 60)), "20d 18h"); +} diff --git a/crates/statsai/src/cli/tests/support.rs b/crates/statsai/src/cli/tests/support.rs index f084db6..b119cdd 100644 --- a/crates/statsai/src/cli/tests/support.rs +++ b/crates/statsai/src/cli/tests/support.rs @@ -106,6 +106,8 @@ pub(super) fn test_sync_command(sink: &str) -> SyncCommand { exclude_tasks: false, include_activity: false, exclude_activity: false, + include_sessions: false, + exclude_sessions: false, } } @@ -229,6 +231,7 @@ pub(super) fn test_event( started_at, ended_at: None, duration_seconds: None, + turn_started_at: None, }, model: None, usage: UsageCounts { diff --git a/crates/statsai/src/cli/tests/sync/batch.rs b/crates/statsai/src/cli/tests/sync/batch.rs index 5d51eff..efe7405 100644 --- a/crates/statsai/src/cli/tests/sync/batch.rs +++ b/crates/statsai/src/cli/tests/sync/batch.rs @@ -1508,6 +1508,7 @@ fn sync_batch_serialization_excludes_local_task_entities() { account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: Vec::new(), @@ -1612,6 +1613,7 @@ fn build_sync_batch_respects_project_and_task_opt_ins() { include_projects: true, include_tasks: false, include_activity: false, + include_sessions: false, }) .expect("persist sync preferences"); let (persisted_batch, persisted_mode) = @@ -1703,6 +1705,7 @@ fn code_change_metric_project_ids_follow_sync_project_preferences() { include_projects: true, include_tasks: false, include_activity: false, + include_sessions: false, }) .expect("persist project opt-in"); let exclude_command = SyncCommand { @@ -2094,6 +2097,7 @@ fn activity_sync_payload_omits_paths_commands_and_invocation_ids() { let include_command = SyncCommand { include_activity: true, + include_sessions: false, ..test_sync_command("file") }; let include_target = sync_target(&include_command).expect("target"); @@ -2129,3 +2133,70 @@ fn activity_sync_payload_omits_paths_commands_and_invocation_ids() { assert!(excluded.get("activity_rollups").is_none()); assert!(excluded.get("activity_coverage").is_none()); } + +#[test] +fn excluding_projects_retires_hosted_sessions_and_excluding_sessions_keeps_them() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/codex-sessions-exclude-projects"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + store + .insert_event(&test_event( + "codex", + &source, + Utc::now(), + None, + TokenParts::total(15), + )) + .expect("event"); + let command = |exclude_projects: bool, exclude_sessions: bool| SyncCommand { + exclude_projects, + exclude_sessions, + dry_run: true, + endpoint: Some("https://api.example.com/api/sync/batches".to_string()), + ..test_sync_command("http") + }; + let target = sync_target(&command(false, false)).expect("target"); + let snapshot_session_ids = |command: &SyncCommand| { + build_sync_batch(command, &store, "device", &target) + .expect("batch") + .0 + .authoritative_snapshot + .and_then(|snapshot| snapshot.session_rollup_ids) + }; + + // Nothing hosted yet: excluding projects has no sessions to retire. + assert_eq!(snapshot_session_ids(&command(true, false)), None); + + let now = Utc + .with_ymd_and_hms(2026, 6, 20, 12, 0, 0) + .single() + .expect("date"); + let mut synced = test_task_only_sync_batch(now, 0, 0); + synced.batch_id = "batch_sessions".to_string(); + synced.sessions = test_session_rollups(2); + let logical_batch_id = logical_http_rollup_batch_id(&synced.batch_id).to_string(); + record_rollup_sync_chunk_success(&store, "http", &target, &logical_batch_id, &synced) + .expect("record synced sessions"); + + // Hosted sessions carry their project, so excluding projects retires them. + assert_eq!( + snapshot_session_ids(&command(true, false)), + Some(Vec::new()) + ); + // Turning off only session sync leaves them in place. + store + .set_sync_preferences(SyncPreferences { + include_projects: true, + include_tasks: false, + include_activity: false, + include_sessions: true, + }) + .expect("sessions on"); + assert_eq!(snapshot_session_ids(&command(false, true)), None); +} diff --git a/crates/statsai/src/cli/tests/sync/chunking/budget.rs b/crates/statsai/src/cli/tests/sync/chunking/budget.rs index 21e008a..bbc89eb 100644 --- a/crates/statsai/src/cli/tests/sync/chunking/budget.rs +++ b/crates/statsai/src/cli/tests/sync/chunking/budget.rs @@ -48,6 +48,7 @@ fn http_rollup_sync_proactively_splits_batches_to_fit_d1_budget() { account_evidence_summaries: vec![], activity_rollups: vec![], activity_coverage: vec![], + sessions: vec![], events: vec![], summaries, task_buckets: vec![], @@ -201,6 +202,7 @@ fn http_rollup_project_counts_include_path_only_projects() { account_evidence_summaries: vec![], activity_rollups: vec![], activity_coverage: vec![], + sessions: vec![], events: vec![], summaries: vec![summary], task_buckets: vec![], @@ -304,6 +306,7 @@ fn test_dense_task_only_sync_batch(now: DateTime, span_count: usize) -> Syn account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: vec![TaskBucketSnapshot { @@ -505,6 +508,7 @@ fn test_multi_bucket_dense_task_only_sync_batch( account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets, @@ -552,3 +556,96 @@ fn multi_bucket_dense_task_sync_splits_to_fit_chunked_write_budget() { .iter() .all(|chunk| estimate_http_rollup_d1_queries(chunk) <= HTTP_ROLLUP_D1_QUERY_BUDGET)); } + +#[test] +fn session_project_and_activity_rows_match_the_backend_d1_cost() { + let now = Utc + .with_ymd_and_hms(2026, 6, 20, 12, 0, 0) + .single() + .expect("date"); + let baseline = test_task_only_sync_batch(now, 0, 0); + let baseline_queries = estimate_http_rollup_d1_queries(&baseline); + + let mut bare_session = baseline.clone(); + let mut sessions = test_session_rollups(1); + sessions[0].project = None; + sessions[0].provider_account_id = None; + bare_session.sessions = sessions; + assert_eq!( + estimate_http_rollup_d1_queries(&bare_session), + baseline_queries + 4, + "session lookup, write, ownership, and dashboard timestamp" + ); + + let mut named_session = baseline.clone(); + named_session.sessions = test_session_rollups(1); + assert_eq!(http_rollup_project_count(&named_session), 1); + assert_eq!(http_rollup_project_location_count(&named_session), 1); + assert_eq!( + estimate_http_rollup_d1_queries(&named_session), + baseline_queries + 8, + "project, location, location lookup, and account-alias lookup" + ); + + let mut shared_project = named_session.clone(); + let mut shared_sessions = test_session_rollups(2); + shared_sessions[1].project = shared_sessions[0].project.clone(); + shared_project.sessions = shared_sessions; + assert_eq!(http_rollup_project_count(&shared_project), 1); + assert_eq!(http_rollup_project_location_count(&shared_project), 1); + assert_eq!( + estimate_http_rollup_d1_queries(&shared_project), + estimate_http_rollup_d1_queries(&named_session) + ); + + let mut activity = baseline.clone(); + activity.activity_rollups = vec![test_activity_rollup(0, None)]; + assert_eq!( + estimate_http_rollup_d1_queries(&activity), + baseline_queries + 3, + "activity write, ownership, and dashboard timestamp" + ); + activity.activity_rollups = vec![test_activity_rollup( + 0, + Some(provider_account_id("codex", "personal")), + )]; + assert_eq!( + estimate_http_rollup_d1_queries(&activity), + baseline_queries + 4, + "activity rows with a provider account also pay for the alias lookup" + ); + activity.activity_coverage = vec![test_activity_coverage(0)]; + assert_eq!( + estimate_http_rollup_d1_queries(&activity), + baseline_queries + 6, + "coverage is its own write and ownership; the timestamp and alias stay one query" + ); +} + +#[test] +fn session_batches_with_many_projects_split_under_the_d1_budget() { + let now = Utc + .with_ymd_and_hms(2026, 6, 20, 12, 0, 0) + .single() + .expect("date"); + let mut batch = test_task_only_sync_batch(now, 0, 0); + batch.batch_id = "batch_session_projects".to_string(); + batch.sessions = test_session_rollups(40); + assert_eq!(http_rollup_project_count(&batch), 40); + assert_eq!(http_rollup_project_location_count(&batch), 40); + assert!(estimate_http_rollup_d1_queries(&batch) > HTTP_ROLLUP_D1_QUERY_BUDGET); + + let chunks = split_http_rollup_sync_batches(&batch); + + assert!(chunks.len() > 1); + assert_eq!( + chunks + .iter() + .map(|chunk| chunk.sessions.len()) + .sum::(), + batch.sessions.len() + ); + assert!(chunks + .iter() + .all(|chunk| estimate_http_rollup_d1_queries(chunk) <= HTTP_ROLLUP_D1_QUERY_BUDGET)); +} diff --git a/crates/statsai/src/cli/tests/sync/chunking/mod.rs b/crates/statsai/src/cli/tests/sync/chunking/mod.rs index a1bd939..e32ddec 100644 --- a/crates/statsai/src/cli/tests/sync/chunking/mod.rs +++ b/crates/statsai/src/cli/tests/sync/chunking/mod.rs @@ -41,6 +41,7 @@ fn http_rollup_sync_splits_large_summary_batches() { account_evidence_summaries: vec![], activity_rollups: vec![], activity_coverage: vec![], + sessions: vec![], events: vec![], summaries, task_buckets: vec![], @@ -94,6 +95,7 @@ fn http_rollup_sync_sends_authoritative_snapshot_after_data_chunks() { account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: Vec::new(), @@ -150,6 +152,7 @@ fn http_rollup_sync_bounds_authoritative_snapshot_chunks() { account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: Vec::new(), @@ -274,6 +277,7 @@ fn http_rollup_sync_splits_metadata_away_from_summaries() { account_evidence_summaries: vec![], activity_rollups: vec![], activity_coverage: vec![], + sessions: vec![], events: vec![], summaries, task_buckets: vec![], @@ -346,6 +350,7 @@ fn http_rollup_metadata_budget_retries_preserve_all_metadata_kinds() { account_evidence_summaries: vec![], activity_rollups: vec![], activity_coverage: vec![], + sessions: vec![], events: vec![], summaries: vec![], task_buckets: vec![], @@ -382,3 +387,69 @@ fn http_rollup_metadata_budget_retries_preserve_all_metadata_kinds() { assert!(chunks.iter().any(|chunk| !chunk.sources.is_empty())); assert!(chunks.iter().any(|chunk| !chunk.accounts.is_empty())); } + +#[test] +fn http_rollup_snapshot_preserves_opt_in_session_retirement() { + let now = Utc + .with_ymd_and_hms(2026, 5, 29, 10, 12, 43) + .single() + .expect("date"); + let empty = SyncBatch { + schema_version: SYNC_BATCH_SCHEMA_VERSION.to_string(), + batch_id: "batch_empty_sessions".to_string(), + device_id: "device".to_string(), + sources: Vec::new(), + accounts: Vec::new(), + source_account_assignments: Vec::new(), + subscriptions: Vec::new(), + account_plan_observations: Vec::new(), + account_evidence_summaries: Vec::new(), + activity_rollups: Vec::new(), + activity_coverage: Vec::new(), + sessions: Vec::new(), + events: Vec::new(), + summaries: Vec::new(), + task_buckets: Vec::new(), + task_verifications: Vec::new(), + code_change_metrics: Vec::new(), + quota_cycle_contributions: Vec::new(), + authoritative_snapshot: Some(SyncAuthoritativeSnapshot { + session_rollup_ids: Some(Vec::new()), + ..SyncAuthoritativeSnapshot::default() + }), + created_at: now, + }; + let empty_snapshot = split_http_rollup_sync_batches(&empty) + .into_iter() + .find_map(|chunk| chunk.authoritative_snapshot) + .expect("snapshot chunk"); + assert_eq!(empty_snapshot.session_rollup_ids.as_deref(), Some(&[][..])); + let json = serde_json::to_value(&empty_snapshot).expect("serialize"); + assert_eq!(json["session_rollup_ids"], serde_json::json!([])); + + let ids = (0..=HTTP_ROLLUP_SNAPSHOT_IDS_PER_BATCH) + .map(|index| format!("session-{index}")) + .collect::>(); + let split = SyncBatch { + batch_id: "batch_split_sessions".to_string(), + authoritative_snapshot: Some(SyncAuthoritativeSnapshot { + session_rollup_ids: Some(ids.clone()), + ..SyncAuthoritativeSnapshot::default() + }), + ..empty + }; + let parts = split_http_rollup_sync_batches(&split) + .into_iter() + .filter_map(|chunk| chunk.authoritative_snapshot) + .collect::>(); + assert!(parts.len() > 1); + let seen = parts + .iter() + .flat_map(|part| { + part.session_rollup_ids + .clone() + .expect("parts that carry session ids keep the retirement key") + }) + .collect::>(); + assert_eq!(seen, ids); +} diff --git a/crates/statsai/src/cli/tests/sync/chunking/payloads.rs b/crates/statsai/src/cli/tests/sync/chunking/payloads.rs index 89416ea..80c1765 100644 --- a/crates/statsai/src/cli/tests/sync/chunking/payloads.rs +++ b/crates/statsai/src/cli/tests/sync/chunking/payloads.rs @@ -37,6 +37,7 @@ fn test_quota_only_sync_batch(now: DateTime, count: usize) -> SyncBatch { account_evidence_summaries: vec![], activity_rollups: vec![], activity_coverage: vec![], + sessions: vec![], events: vec![], summaries: vec![], task_buckets: vec![], @@ -214,6 +215,72 @@ fn http_rollup_retry_splits_code_change_only_payloads() { ); } +#[test] +fn http_rollup_retry_splits_sessions_and_activity_after_d1_budget_rejection() { + let now = Utc + .with_ymd_and_hms(2026, 6, 20, 12, 0, 0) + .single() + .expect("date"); + let budget_error = anyhow::anyhow!( + r#"sync endpoint returned HTTP 413: {{"error":"sync_batch_d1_query_budget_exceeded","estimatedQueries":70,"maxQueries":45}}"# + ); + let mut sessions = test_task_only_sync_batch(now, 0, 0); + sessions.sessions = test_session_rollups(2); + assert!(should_retry_http_rollup_chunk_after_error( + &sessions, + &budget_error + )); + let session_chunks = split_http_rollup_sync_batch_after_budget_error(&sessions); + assert_eq!(session_chunks.len(), 2); + assert!(session_chunks.iter().all(|chunk| chunk.sessions.len() == 1)); + + let mut one_session = sessions.clone(); + one_session.sessions.truncate(1); + assert!(!should_retry_http_rollup_chunk_after_error( + &one_session, + &budget_error + )); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/codex-http-session-mixed"), + LocationOrigin::Configured, + ); + one_session.summaries = vec![test_summary("codex", &source, now, 10, None)]; + assert!(should_retry_http_rollup_chunk_after_error( + &one_session, + &budget_error + )); + + let mut activity = test_task_only_sync_batch(now, 0, 0); + activity.activity_rollups = vec![test_activity_rollup(0, None), test_activity_rollup(1, None)]; + assert!(should_retry_http_rollup_chunk_after_error( + &activity, + &budget_error + )); + let activity_chunks = split_http_rollup_sync_batch_after_budget_error(&activity); + assert_eq!(activity_chunks.len(), 2); + assert!(activity_chunks + .iter() + .all(|chunk| chunk.activity_rollups.len() == 1)); + + let mut coverage = test_task_only_sync_batch(now, 0, 0); + coverage.activity_coverage = vec![test_activity_coverage(0), test_activity_coverage(1)]; + assert!(should_retry_http_rollup_chunk_after_error( + &coverage, + &budget_error + )); + + let mut mixed_activity = test_task_only_sync_batch(now, 0, 0); + mixed_activity.activity_rollups = vec![test_activity_rollup(0, None)]; + mixed_activity.summaries = vec![test_summary("codex", &source, now, 10, None)]; + assert!(should_retry_http_rollup_chunk_after_error( + &mixed_activity, + &budget_error + )); +} + #[test] fn http_rollup_retry_halves_task_only_bucket_chunks() { let now = Utc @@ -264,6 +331,7 @@ fn http_rollup_sends_metadata_before_task_chunks() { account_evidence_summaries: vec![], activity_rollups: vec![], activity_coverage: vec![], + sessions: vec![], events: vec![], summaries: vec![], task_buckets: test_task_only_sync_batch(now, 1, 0).task_buckets, diff --git a/crates/statsai/src/cli/tests/sync/http.rs b/crates/statsai/src/cli/tests/sync/http.rs index d6d1d7a..9a4dc6a 100644 --- a/crates/statsai/src/cli/tests/sync/http.rs +++ b/crates/statsai/src/cli/tests/sync/http.rs @@ -55,6 +55,7 @@ fn http_rollup_sync_retries_smaller_batches_after_budget_rejection() { account_evidence_summaries: vec![], activity_rollups: vec![], activity_coverage: vec![], + sessions: vec![], events: vec![], summaries, task_buckets: vec![], @@ -111,6 +112,50 @@ fn http_rollup_sync_retries_smaller_batches_after_budget_rejection() { assert!(pending.is_empty()); } +#[test] +fn http_rollup_sync_retries_sessions_after_d1_budget_rejection() { + let store = Store::in_memory().expect("store"); + let endpoint = "https://api.example.com/api/sync/batches".to_string(); + let now = Utc + .with_ymd_and_hms(2026, 6, 20, 12, 0, 0) + .single() + .expect("date"); + let mut batch = test_task_only_sync_batch(now, 0, 0); + batch.batch_id = "batch_sessions".to_string(); + batch.sessions = test_session_rollups(4); + let logical_batch_id = logical_http_rollup_batch_id(&batch.batch_id).to_string(); + let observed = Arc::new(Mutex::new(Vec::new())); + let observed_for_send = Arc::clone(&observed); + + send_http_rollup_chunk_with_retry_using(&batch, &|chunk| { + observed_for_send + .lock() + .expect("observed lock") + .push((chunk.batch_id.clone(), chunk.sessions.len())); + if chunk.sessions.len() > 2 { + return Err(anyhow::Error::msg( + r#"sync endpoint returned HTTP 413: {"error":"sync_batch_d1_query_budget_exceeded","estimatedQueries":70,"maxQueries":45}"#, + )); + } + record_rollup_sync_chunk_success(&store, "http", &endpoint, &logical_batch_id, chunk) + }) + .expect("send"); + + let observed = observed.lock().expect("observed lock").clone(); + assert_eq!( + observed, + vec![ + ("batch_sessions".to_string(), 4), + ("batch_sessions_sessions_1".to_string(), 2), + ("batch_sessions_sessions_2".to_string(), 2), + ] + ); + let pending = store + .pending_session_rollups_for_sync("http", &endpoint, &batch.sessions) + .expect("pending sessions"); + assert!(pending.is_empty()); +} + #[test] fn http_rollup_sync_retries_smaller_batches_after_payload_too_large() { let store = Store::in_memory().expect("store"); @@ -152,6 +197,7 @@ fn http_rollup_sync_retries_smaller_batches_after_payload_too_large() { account_evidence_summaries: vec![], activity_rollups: vec![], activity_coverage: vec![], + sessions: vec![], events: vec![], summaries, task_buckets: vec![], diff --git a/crates/statsai/src/cli/tests/sync/mod.rs b/crates/statsai/src/cli/tests/sync/mod.rs index d29e048..9ac0fcd 100644 --- a/crates/statsai/src/cli/tests/sync/mod.rs +++ b/crates/statsai/src/cli/tests/sync/mod.rs @@ -308,6 +308,7 @@ pub(crate) fn test_task_only_sync_batch( account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets, @@ -319,6 +320,102 @@ pub(crate) fn test_task_only_sync_batch( } } +pub(crate) fn test_session_rollups(count: usize) -> Vec { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/codex-http-session-budget"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let now = Utc + .with_ymd_and_hms(2026, 6, 20, 12, 0, 0) + .single() + .expect("date"); + let account_id = provider_account_id("codex", "personal"); + for index in 0..count { + let mut event = test_event( + "codex", + &source, + now + Duration::minutes(index as i64), + Some(account_id.clone()), + TokenParts::total(10), + ); + event.session.session_id = format!("session-budget-{index}"); + event.project = Some(ProjectInfo { + project_id: format!("project-budget-{index}"), + project_label: Some(format!("Project {index}")), + repo_remote_hash: Some(format!("repo-hash-{index}")), + repo_label: Some(format!("owner/repo-{index}")), + branch_hash: None, + branch_label: None, + path_hash: Some(format!("path-hash-{index}")), + path_label: Some(format!("/tmp/project-{index}")), + }); + store.insert_event(&event).expect("event"); + } + let rollups = store.all_session_rollups().expect("session rollups"); + assert_eq!(rollups.len(), count); + rollups +} + +pub(crate) fn test_activity_rollup( + index: usize, + provider_account_id: Option, +) -> statsai_core::ActivityRollupV1 { + let seen = Utc + .with_ymd_and_hms(2026, 6, 20, 12, 0, 0) + .single() + .expect("date"); + statsai_core::ActivityRollupV1 { + schema_version: statsai_core::ACTIVITY_ROLLUP_SCHEMA_VERSION.to_string(), + rollup_id: format!("activity-rollup-{index}"), + device_id: "device".to_string(), + source_id: SourceId("source".to_string()), + provider: "codex".to_string(), + provider_account_id, + day: "2026-06-20".to_string(), + kind: statsai_core::ActivityKind::Tool, + entity_key: "shell".to_string(), + display_name: "shell".to_string(), + family: statsai_core::ActivityFamily::Shell, + mcp_server: None, + mcp_tool: None, + plugin: None, + skill_catalog: None, + model: None, + calls: 1, + succeeded: 1, + failed: 0, + unknown: 0, + duration_samples: 1, + duration_sum_ms: 10, + duration_max_ms: Some(10), + duration_kind: Some(statsai_core::ActivityDurationKind::Reported), + first_seen: seen, + last_seen: seen, + evidence: "codex-native-items".to_string(), + } +} + +pub(crate) fn test_activity_coverage(index: usize) -> statsai_core::ActivityCoverageV1 { + statsai_core::ActivityCoverageV1 { + schema_version: statsai_core::ACTIVITY_COVERAGE_SCHEMA_VERSION.to_string(), + coverage_id: format!("activity-coverage-{index}"), + device_id: "device".to_string(), + source_id: SourceId("source".to_string()), + provider: "codex".to_string(), + day: "2026-06-20".to_string(), + day_end: "2026-06-20".to_string(), + kind: statsai_core::ActivityKind::Tool, + level: statsai_core::ActivityCoverageLevel::Complete, + evidence: "codex-native-items".to_string(), + parser_revision: statsai_core::ACTIVITY_PARSER_REVISION.to_string(), + } +} + pub(crate) fn test_code_change_metric( index: usize, now: DateTime, diff --git a/crates/statsai/src/lib.rs b/crates/statsai/src/lib.rs index f31a968..bd7852d 100644 --- a/crates/statsai/src/lib.rs +++ b/crates/statsai/src/lib.rs @@ -332,6 +332,7 @@ mod tests { started_at, ended_at: None, duration_seconds: None, + turn_started_at: None, }, model: Some(ModelInfo { name: Some("codex-auto-review".to_string()), diff --git a/crates/statsai/tests/cli_surface/help.txt b/crates/statsai/tests/cli_surface/help.txt index ef70d41..d240415 100644 --- a/crates/statsai/tests/cli_surface/help.txt +++ b/crates/statsai/tests/cli_surface/help.txt @@ -6,6 +6,7 @@ Usage: statsai [OPTIONS] Commands: scan Scan local provider sources for usage events report Show usage reports (weekly, monthly, all-time, or a date range) + sessions List local session rollups source Manage configured source paths account List canonical provider accounts subscription Manage subscription periods @@ -114,6 +115,22 @@ Options: --verbose Show source paths and reasoning tokens --subscriptions Include subscription-value rows -h, --help Print help +===== statsai sessions --help ===== +List local session rollups + +Usage: statsai sessions [OPTIONS] + +Options: + --from Range start (YYYY-MM-DD or RFC3339). Defaults to the last 7 days + --store Path to SQLite store + --device-id Device identifier for multi-device sync + --to Range end (YYYY-MM-DD or RFC3339). Defaults to now + --provider Provider filter + --project Project id, key, or label + --sort Sort descending [default: started] [possible values: started, tokens, cost, duration, messages] + --limit Maximum sessions to print [default: 50] + --json Output as JSON + -h, --help Print help ===== statsai source --help ===== Manage configured source paths @@ -1036,11 +1053,13 @@ Options: --yes Confirm destructive sync reset actions --dry-run Build the sync batch without writing --include-projects Enable project metadata sync for this device and future syncs - --exclude-projects Disable project metadata sync for this device and future syncs + --exclude-projects Disable project metadata sync for this device and future syncs; hosted sessions are retired --include-tasks Enable hosted task sync for this device and future syncs (implies --include-projects) --exclude-tasks Disable hosted task sync for this device and future syncs --include-activity Enable hosted agent activity sync for this device and future syncs --exclude-activity Disable hosted agent activity sync and prune names on the next successful HTTP sync + --include-sessions Enable hosted session sync for this device and future syncs (implies --include-projects) + --exclude-sessions Disable hosted session sync for this device and future syncs -h, --help Print help ===== statsai schema --help ===== Print JSON schemas for backend-facing contracts diff --git a/crates/statsai/tests/cli_surface/schema-sync-batch.json b/crates/statsai/tests/cli_surface/schema-sync-batch.json index 19888d4..2610831 100644 --- a/crates/statsai/tests/cli_surface/schema-sync-batch.json +++ b/crates/statsai/tests/cli_surface/schema-sync-batch.json @@ -86,6 +86,13 @@ "schema_version": { "type": "string" }, + "sessions": { + "description": "Opt-in session rollups. Empty batches omit the key. Each row is `session_rollup.v1` with the closed ingest key set.", + "type": "array", + "items": { + "$ref": "#/definitions/SessionRollupV1" + } + }, "source_account_assignments": { "type": "array", "items": { @@ -1421,6 +1428,40 @@ } } }, + "SessionCostWire": { + "description": "Cost keys accepted on `session_rollup.v1`.", + "type": "object", + "properties": { + "estimated_api_equivalent_micro_usd": { + "type": [ + "integer", + "null" + ], + "format": "int64" + }, + "estimated_api_equivalent_usd": { + "type": [ + "integer", + "null" + ], + "format": "int64" + }, + "provider_reported_micro_usd": { + "type": [ + "integer", + "null" + ], + "format": "int64" + }, + "provider_reported_usd": { + "type": [ + "integer", + "null" + ], + "format": "int64" + } + } + }, "SessionInfo": { "type": "object", "required": [ @@ -1461,6 +1502,294 @@ "string", "null" ] + }, + "turn_started_at": { + "description": "When the prompt that started this event's turn was sent, for providers that record one timestamp per message. The session's active time runs from here to the event.", + "type": [ + "string", + "null" + ], + "format": "date-time" + } + } + }, + "SessionModelWire": { + "description": "One model row on `session_rollup.v1`.", + "type": "object", + "required": [ + "cost", + "model", + "usage" + ], + "properties": { + "cost": { + "$ref": "#/definitions/SessionCostWire" + }, + "metrics": { + "anyOf": [ + { + "$ref": "#/definitions/SummaryModelMetrics" + }, + { + "type": "null" + } + ] + }, + "model": { + "$ref": "#/definitions/ModelInfo" + }, + "usage": { + "$ref": "#/definitions/SessionUsageWire" + } + } + }, + "SessionRollupV1": { + "description": "One local session, aggregated from the usage events that share its hashed id.\n\n`session_id` is the hashed `session_…` identity already stored on each event. Raw provider session ids never appear on this record.", + "type": "object", + "required": [ + "cost", + "device_id", + "ended_at", + "models", + "provider", + "requests", + "schema_version", + "session_id", + "source_id", + "started_at", + "updated_at", + "usage" + ], + "properties": { + "active_seconds": { + "description": "Time the agent was working: the union of its turns, each from the prompt until the agent's last message, as the provider recorded them. Idle time between turns does not count. `None` when no turn times were recorded.", + "default": null, + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "assistant_messages": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "cost": { + "description": "Closed cost object. Currency, pricing source, and confidence stay local.", + "allOf": [ + { + "$ref": "#/definitions/SessionCostWire" + } + ] + }, + "developer_messages": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "device_id": { + "type": "string" + }, + "duration_seconds": { + "description": "Wall-clock span from the first event to the last, idle time included. `None` when the session recorded only a start. Unknown durations go out as null; ingest stores them without an end so they stay out of medians.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "ended_at": { + "type": "string", + "format": "date-time" + }, + "models": { + "type": "array", + "items": { + "$ref": "#/definitions/SessionModelWire" + } + }, + "primary_model": { + "type": [ + "string", + "null" + ] + }, + "project": { + "anyOf": [ + { + "$ref": "#/definitions/ProjectInfo" + }, + { + "type": "null" + } + ] + }, + "provider": { + "type": "string" + }, + "provider_account_id": { + "type": [ + "string", + "null" + ] + }, + "requests": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "schema_version": { + "type": "string" + }, + "session_id": { + "type": "string" + }, + "source_id": { + "type": "string" + }, + "started_at": { + "type": "string", + "format": "date-time" + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "title_source": { + "anyOf": [ + { + "$ref": "#/definitions/SessionTitleSource" + }, + { + "type": "null" + } + ] + }, + "total_messages": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "updated_at": { + "type": "string", + "format": "date-time" + }, + "usage": { + "description": "Closed usage object. Cache-lifetime splits stay local; the ingest allowlist does not include them.", + "allOf": [ + { + "$ref": "#/definitions/SessionUsageWire" + } + ] + }, + "user_messages": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + } + } + }, + "SessionTitleSource": { + "description": "Where a session title was resolved from. Event titles win over task spans, which win over archived conversations.", + "type": "string", + "enum": [ + "event", + "task_span", + "archive" + ] + }, + "SessionUsageWire": { + "description": "Usage keys accepted on `session_rollup.v1`. Five-minute and one-hour cache splits are folded into `cache_creation_tokens` before this leaves the device.", + "type": "object", + "properties": { + "cache_creation_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "cache_read_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "input_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "local_eval_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "local_prompt_eval_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "output_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "reasoning_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "requests": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "total_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 } } }, @@ -2055,6 +2384,16 @@ "type": "string" } }, + "session_rollup_ids": { + "description": "Present only when this collector is opting into session retirement. An empty array prunes every hosted session for the device. Absence leaves hosted sessions alone, including on pre-session v6 collectors.", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + } + }, "snapshot_id": { "type": "string" }, diff --git a/crates/statsai/tests/report_cli.rs b/crates/statsai/tests/report_cli.rs index 6b3ad66..b139515 100644 --- a/crates/statsai/tests/report_cli.rs +++ b/crates/statsai/tests/report_cli.rs @@ -60,6 +60,7 @@ fn test_event( started_at, ended_at: None, duration_seconds: None, + turn_started_at: None, }, model: None, usage: UsageCounts { diff --git a/docs/sync-contract.md b/docs/sync-contract.md index 4662c13..8dd6241 100644 --- a/docs/sync-contract.md +++ b/docs/sync-contract.md @@ -23,6 +23,24 @@ names leave the device only when the collector preference `include_activity` is on; turning it off sends an authoritative empty activity ID set so the backend prunes hosted names. Invocation IDs, arguments, outputs, commands, prompts, and file paths never appear in a v6 payload. +The same batch schema carries opt-in `sessions` rows. Each row is +`session_rollup.v1` with a closed key set: hashed session id, provider, +source, account, time range, duration, usage, requests, cost, models, message +counts, project, bounded title, and `title_source` (`event`, `task_span`, or +`archive`). `duration_seconds` is the wall-clock span and is null when a +session recorded only a start; the backend must accept null before a collector +sends it. `active_seconds` is the time the agent was working, the union of the +turns the provider recorded, and is null when none were; collectors before it +omit the key. Events keyed by their +own provider record, such as Cursor's usage-export rows, belong to no session +and are not sent, and neither is a session with no project and no title. Prompts, responses, paths, and raw provider session ids stay on +the device. Session retirement is not implied by `sync_batch.v6`. A snapshot +retires hosted sessions only when it includes `session_rollup_ids`. An empty +array prunes them. Omitting the key leaves hosted sessions in place, which is +what the collector does when `include_sessions` alone is off. With projects +excluded, it sends the empty array until the target holds none of its sessions, +since each hosted session carries a project. Pre-v6 acknowledgements +omit the `sessions` counter. The collector owns local scanning, normalization, idempotent local storage, and privacy scrubbing. The backend owns authentication, validation, deduplication, rollups, and user-facing queries. The production path sends sanitized batches to @@ -273,7 +291,8 @@ observations and derives its own cycles from them, so acknowledging another device's cycles would tell the sender they had been stored when they had not. It likewise rejects `activity_rollups` and `activity_coverage`: hosted activity is a Cloudflare-only collection, and the loopback daemon has nowhere to store -another device's names. +another device's names. It rejects `sessions` for the same reason: hosted +session rollups are stored only by `/api/sync/batches`. `/api/sync/batches` is the production contract. A compatible backend should: - require an authenticated device access token @@ -289,7 +308,8 @@ another device's names. device; v3 fragments also carry code-change metric IDs; v4 fragments also carry quota-cycle contribution IDs; v5 fragments also carry account-plan observation and evidence-summary IDs; v6 fragments also carry activity - rollup and coverage IDs; each fragment carries zero-based `part_index` and a + rollup and coverage IDs and, when the collector opts into session retirement, + `session_rollup_ids`; each fragment carries zero-based `part_index` and a common `part_count`, with at most 200 IDs across its ID arrays - stage snapshot ownership without pruning until the final in-order fragment; then apply ownership and deletion reconciliation atomically, pruning a hosted @@ -359,7 +379,8 @@ run immediately. `sync_batch.v5` returns `sync_ack.v5`, which additionally adds the `account_plan_observations` and `account_evidence_summaries` counters. `sync_batch.v6` returns `sync_ack.v6`, which additionally adds the -`activity_rollups` and `activity_coverage` counters. +`activity_rollups`, `activity_coverage`, and `sessions` counters. Earlier +acknowledgements omit `sessions`. Collectors require the acknowledgement version to match the submitted batch version exactly; a v1 acknowledgement cannot successfully acknowledge a v2 batch, a v2 acknowledgement cannot acknowledge a v3 batch, a v3 @@ -489,7 +510,7 @@ verification actions for `sync_batch.v2` and later, plus code-change metrics for `sync_batch.v3` and later, plus quota-cycle contributions for `sync_batch.v4` and later, plus privacy-safe account-plan evidence for `sync_batch.v5`, plus opt-in activity rollups and coverage for -`sync_batch.v6`. The collector now prepares those +`sync_batch.v6`, plus opt-in session rollups on that same v6 schema. The collector now prepares those daily rollups before HTTP sync, so a normal Cloudflare sync can populate the dashboard without shipping raw events. Repeated batches are idempotent by stable IDs.