From d2f87ee476eb7f0b06728424963cd989cb844da9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 25 Sep 2026 19:18:01 +0000 Subject: [PATCH 01/16] Add local session rollups and a sessions command Aggregate hashed sessions into session_rollup.v1, count Claude and OpenCode messages, and list them with statsai sessions. Co-authored-by: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> --- crates/statsai-adapters/src/claude/parse.rs | 64 +- .../src/claude/tests/parse.rs | 55 +- crates/statsai-adapters/src/event.rs | 26 + crates/statsai-adapters/src/lib.rs | 8 +- crates/statsai-adapters/src/opencode/mod.rs | 55 +- crates/statsai-core/src/tests/types.rs | 77 ++ crates/statsai-core/src/types/mod.rs | 2 + crates/statsai-core/src/types/session.rs | 48 + .../src/account_plan/observations.rs | 7 +- crates/statsai-store/src/archive/write.rs | 5 + crates/statsai-store/src/events.rs | 87 +- crates/statsai-store/src/lib.rs | 7 + crates/statsai-store/src/migrations/mod.rs | 19 +- crates/statsai-store/src/migrations/v2.rs | 32 +- crates/statsai-store/src/pricing/mod.rs | 6 +- .../src/pricing/tests/repricing.rs | 2 +- crates/statsai-store/src/rollups/mod.rs | 3 + crates/statsai-store/src/rollups/sessions.rs | 922 ++++++++++++++++++ crates/statsai-store/src/tasks/buckets.rs | 15 + crates/statsai-store/src/tasks/spans.rs | 8 + crates/statsai-store/src/tests/rollups/mod.rs | 1 + .../src/tests/rollups/sessions.rs | 327 +++++++ crates/statsai/src/cli/args/mod.rs | 6 +- crates/statsai/src/cli/args/sessions.rs | 43 + crates/statsai/src/cli/dispatch.rs | 2 + crates/statsai/src/cli/mod.rs | 2 + crates/statsai/src/cli/sessions.rs | 151 +++ crates/statsai/src/cli/tests/mod.rs | 1 + crates/statsai/src/cli/tests/sessions.rs | 100 ++ crates/statsai/tests/cli_surface/help.txt | 17 + 30 files changed, 2034 insertions(+), 64 deletions(-) create mode 100644 crates/statsai-core/src/types/session.rs create mode 100644 crates/statsai-store/src/rollups/sessions.rs create mode 100644 crates/statsai-store/src/tests/rollups/sessions.rs create mode 100644 crates/statsai/src/cli/args/sessions.rs create mode 100644 crates/statsai/src/cli/sessions.rs create mode 100644 crates/statsai/src/cli/tests/sessions.rs diff --git a/crates/statsai-adapters/src/claude/parse.rs b/crates/statsai-adapters/src/claude/parse.rs index 67fddb5..53a88c5 100644 --- a/crates/statsai-adapters/src/claude/parse.rs +++ b/crates/statsai-adapters/src/claude/parse.rs @@ -5,12 +5,13 @@ use crate::model::{ }; use crate::ProviderAdapter; use crate::{ - fallback_session_id, file_modified_timestamp, infer_missing_output, metadata_only_privacy, - number_at_any, project_context_from_path_fallback, push_deduped, read_bounded_jsonl_line, - resolve_project_context, resolve_project_context_cached, stats_cache_date_end, - timestamp_from_nested_value, timestamp_from_scalar, usage_event, usd_to_micro_usd, - value_as_u64, AdapterScan, BoundedLineRead, DuplicateSelection, EventDeduplication, - FileParseContext, ProjectContextCache, ProviderEventParts, ScanOptions, MAX_JSONL_RECORD_BYTES, + fallback_session_id, file_modified_timestamp, infer_missing_output, message_count_runtime, + metadata_only_privacy, number_at_any, project_context_from_path_fallback, push_deduped, + read_bounded_jsonl_line, resolve_project_context, resolve_project_context_cached, + stats_cache_date_end, timestamp_from_nested_value, timestamp_from_scalar, usage_event, + usd_to_micro_usd, value_as_u64, AdapterScan, BoundedLineRead, DuplicateSelection, + EventDeduplication, FileParseContext, ProjectContextCache, ProviderEventParts, ScanOptions, + MAX_JSONL_RECORD_BYTES, }; use anyhow::{Context, Result}; use chrono::Utc; @@ -43,6 +44,10 @@ pub(crate) fn parse_claude_file( let mut line_bytes = Vec::new(); let mut index = 0usize; + let mut pending_user_messages = HashMap::::new(); + // Later snapshots of one provider record replace the event. User lines + // counted on the first snapshot have to ride along with that replacement. + let mut last_usage_users = HashMap::::new(); loop { let line_status = read_bounded_jsonl_line(&mut reader, &mut line_bytes, MAX_JSONL_RECORD_BYTES)?; @@ -76,18 +81,43 @@ pub(crate) fn parse_claude_file( if reasoning.raw.is_some() { current_reasoning = reasoning; } + let session_raw = claude_session_raw(&value, path); let Some(usage_value) = value .pointer("/message/usage") .or_else(|| value.get("usage")) else { + if claude_record_is_user(&value) { + *pending_user_messages.entry(session_raw).or_default() += 1; + } continue; }; ctx.scan.diagnostics.candidate_usage_rows += 1; let usage = claude_usage_counts_from_value(usage_value); if usage.computed_total() == 0 { + if claude_record_is_user(&value) { + *pending_user_messages.entry(session_raw).or_default() += 1; + } ctx.scan.diagnostics.skipped_zero_events += 1; continue; } + let record_id = claude_provider_record_id(&value); + let fresh_users = pending_user_messages.remove(&session_raw).unwrap_or(0); + let user_messages = match record_id.as_deref() { + Some(record_id) => { + let carried = last_usage_users + .get(&session_raw) + .filter(|(previous, _)| previous == record_id) + .map(|(_, users)| *users) + .unwrap_or(0); + let users = carried.saturating_add(fresh_users); + last_usage_users.insert(session_raw.clone(), (record_id.to_string(), users)); + users + } + None => { + last_usage_users.remove(&session_raw); + fresh_users + } + }; let (timestamp, timestamp_inferred) = timestamp_from_nested_value(&value) .map(|timestamp| (timestamp, false)) .unwrap_or((fallback_timestamp, true)); @@ -106,12 +136,6 @@ pub(crate) fn parse_claude_file( let project = claude_project_context_from_value(&value, indexed_project_metadata, &mut project_cache) .or_else(|| fallback_project.clone()); - let session_raw = value - .get("sessionId") - .or_else(|| value.get("session_id")) - .and_then(Value::as_str) - .map(ToOwned::to_owned) - .unwrap_or_else(|| fallback_session_id(path)); // Claude Code rewrites one streamed response as several records that // share a provider-record identity while cumulative usage grows, so the // last snapshot for that identity is the authoritative one. Records @@ -139,7 +163,7 @@ pub(crate) fn parse_claude_file( duration_seconds: None, model, usage, - runtime: None, + runtime: message_count_runtime(user_messages, 1, 0), session_raw, project, event_kind: "claude_message_usage", @@ -167,6 +191,20 @@ pub(crate) fn parse_claude_file( Ok(()) } +fn claude_session_raw(value: &Value, path: &Path) -> String { + value + .get("sessionId") + .or_else(|| value.get("session_id")) + .and_then(Value::as_str) + .map(ToOwned::to_owned) + .unwrap_or_else(|| fallback_session_id(path)) +} + +fn claude_record_is_user(value: &Value) -> bool { + value.get("type").and_then(Value::as_str) == Some("user") + || value.pointer("/message/role").and_then(Value::as_str) == Some("user") +} + pub(crate) fn claude_provider_record_id(value: &Value) -> Option { let message_id = value .pointer("/message/id") diff --git a/crates/statsai-adapters/src/claude/tests/parse.rs b/crates/statsai-adapters/src/claude/tests/parse.rs index c440efe..7f11232 100644 --- a/crates/statsai-adapters/src/claude/tests/parse.rs +++ b/crates/statsai-adapters/src/claude/tests/parse.rs @@ -498,6 +498,57 @@ fn claude_streaming_snapshots_keep_the_final_usage_for_one_request() { ); } +#[test] +fn claude_user_counts_stay_on_the_kept_streaming_snapshot() { + let dir = tempfile::tempdir().expect("tempdir"); + let projects = dir.path().join("projects"); + std::fs::create_dir_all(&projects).expect("projects"); + let mut file = File::create(projects.join("session.jsonl")).expect("session"); + writeln!( + file, + r#"{{"type":"user","timestamp":"2026-08-05T14:51:09.000Z","sessionId":"session-1","message":{{"role":"user","content":"hello"}}}}"# + ) + .expect("user"); + for (timestamp, uuid, output) in [ + ("2026-08-05T14:51:09.702Z", "record-1", 10), + ("2026-08-05T14:51:11.102Z", "record-2", 40), + ] { + writeln!( + file, + r#"{{"timestamp":"{timestamp}","sessionId":"session-1","uuid":"{uuid}","requestId":"request-1","message":{{"id":"message-1","role":"assistant","model":"claude-opus-5","usage":{{"input_tokens":2,"output_tokens":{output}}}}}}}"# + ) + .expect("snapshot"); + } + writeln!( + file, + r#"{{"type":"user","timestamp":"2026-08-05T14:51:12.000Z","sessionId":"session-1","message":{{"role":"user","content":"again"}}}}"# + ) + .expect("second user"); + writeln!( + file, + r#"{{"timestamp":"2026-08-05T14:51:14.209Z","sessionId":"session-1","uuid":"record-3","requestId":"request-2","message":{{"id":"message-2","role":"assistant","model":"claude-opus-5","usage":{{"input_tokens":3,"output_tokens":5}}}}}}"# + ) + .expect("second request"); + + let source = SourceLocation::local_adapter( + CLAUDE_CODE_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let scan = scan_claude_source(&ClaudeCodeAdapter, &source, &options()).expect("scan"); + + assert_eq!(scan.events.len(), 2); + let first = scan.events[0].runtime.as_ref().expect("first runtime"); + assert_eq!(first.user_messages, Some(1)); + assert_eq!(first.assistant_messages, Some(1)); + assert_eq!(scan.events[0].usage.output_tokens, Some(40)); + let second = scan.events[1].runtime.as_ref().expect("second runtime"); + assert_eq!(second.user_messages, Some(1)); + assert_eq!(second.assistant_messages, Some(1)); +} + #[test] fn claude_streaming_snapshots_with_equal_timestamps_resolve_by_source_line() { let dir = tempfile::tempdir().expect("tempdir"); @@ -1003,7 +1054,9 @@ fn claude_collects_effort_and_effective_speed_but_ignores_service_tier() { .expect("serialize model") .to_string() .contains("service_tier")); - assert!(scan.events[0].runtime.is_none()); + let runtime = scan.events[0].runtime.as_ref().expect("message counts"); + assert_eq!(runtime.assistant_messages, Some(1)); + assert!(runtime.latency_ms.is_none()); } #[test] diff --git a/crates/statsai-adapters/src/event.rs b/crates/statsai-adapters/src/event.rs index c827aa7..d93052c 100644 --- a/crates/statsai-adapters/src/event.rs +++ b/crates/statsai-adapters/src/event.rs @@ -225,6 +225,32 @@ pub(crate) struct ProviderEventParts<'a> { pub(crate) dedupe_salt: Option, } +pub(crate) fn message_count_runtime( + user_messages: u64, + assistant_messages: u64, + developer_messages: u64, +) -> Option { + let total = user_messages + .saturating_add(assistant_messages) + .saturating_add(developer_messages); + if total == 0 { + return None; + } + Some(RuntimeInfo { + runtime_name: None, + host_id: None, + latency_ms: None, + latency_source: None, + time_to_first_token_ms: None, + prompt_eval_duration_ms: None, + eval_duration_ms: None, + total_messages: Some(total), + user_messages: (user_messages > 0).then_some(user_messages), + assistant_messages: (assistant_messages > 0).then_some(assistant_messages), + developer_messages: (developer_messages > 0).then_some(developer_messages), + }) +} + pub(crate) fn usage_event( adapter: &A, source: &SourceLocation, diff --git a/crates/statsai-adapters/src/lib.rs b/crates/statsai-adapters/src/lib.rs index 5924efb..d5ca92a 100644 --- a/crates/statsai-adapters/src/lib.rs +++ b/crates/statsai-adapters/src/lib.rs @@ -54,10 +54,10 @@ pub(crate) use cache::{ scan_candidate_with_compatible_dependencies, ScanCacheNamespaces, }; pub(crate) use event::{ - infer_missing_output, merge_adapter_scan, metadata_only_privacy, metadata_summary, - metric_from_samples, metric_single_sample, push_deduped, subtract_usage_counts, - sum_usage_counts, usage_event, usd_to_micro_usd, DuplicateSelection, EventDedupIndex, - EventDeduplication, MetadataSummaryParts, ProviderEventParts, + infer_missing_output, merge_adapter_scan, message_count_runtime, metadata_only_privacy, + metadata_summary, metric_from_samples, metric_single_sample, push_deduped, + subtract_usage_counts, sum_usage_counts, usage_event, usd_to_micro_usd, DuplicateSelection, + EventDedupIndex, EventDeduplication, MetadataSummaryParts, ProviderEventParts, }; pub(crate) use json::{ file_modified_timestamp, number_at_any, read_bounded_jsonl_line, read_json_file, diff --git a/crates/statsai-adapters/src/opencode/mod.rs b/crates/statsai-adapters/src/opencode/mod.rs index 6dda9e0..37548c0 100644 --- a/crates/statsai-adapters/src/opencode/mod.rs +++ b/crates/statsai-adapters/src/opencode/mod.rs @@ -62,6 +62,7 @@ pub(crate) fn scan_opencode_source( let connection = open_sqlite_readonly(&db_path)?; let todos_by_session = load_opencode_todos(&connection)?; let recovered_session_models = load_opencode_session_models(&connection)?; + let message_role_counts = load_opencode_message_role_counts(&connection)?; let reconstructed_session_ids = recovered_session_models .iter() .filter_map(|(session_id, summary)| { @@ -218,6 +219,9 @@ pub(crate) fn scan_opencode_source( }) .filter(|cost| cost.estimated_micro_usd().is_some()); let priced_from_messages = message_cost.is_some(); + let runtime = message_role_counts.get(&session_id).and_then(|counts| { + message_count_runtime(counts.user, counts.assistant, counts.developer) + }); let mut event = usage_event( adapter, source, @@ -229,7 +233,7 @@ pub(crate) fn scan_opencode_source( duration_seconds, model, usage, - runtime: None, + runtime, session_raw: session_id, project, event_kind: "opencode_session_usage", @@ -499,6 +503,40 @@ pub(crate) fn scan_opencode_source( Ok(scan) } +#[derive(Debug, Clone, Copy, Default)] +struct OpenCodeMessageCounts { + user: u64, + assistant: u64, + developer: u64, +} + +fn load_opencode_message_role_counts( + connection: &Connection, +) -> Result> { + let mut statement = + match connection.prepare("SELECT session_id, json_extract(data, '$.role') FROM message") { + Ok(statement) => statement, + Err(error) if error.to_string().contains("no such table: message") => { + return Ok(HashMap::new()); + } + Err(error) => return Err(error.into()), + }; + let mut rows = statement.query([])?; + let mut counts = HashMap::::new(); + while let Some(row) = rows.next()? { + let session_id: String = row.get(0)?; + let role: Option = row.get(1).ok().flatten(); + let entry = counts.entry(session_id).or_default(); + match role.as_deref() { + Some("user") => entry.user = entry.user.saturating_add(1), + Some("assistant") => entry.assistant = entry.assistant.saturating_add(1), + Some("developer") => entry.developer = entry.developer.saturating_add(1), + _ => {} + } + } + Ok(counts) +} + pub(crate) fn load_opencode_session_models( connection: &Connection, ) -> Result> { @@ -753,6 +791,7 @@ pub(crate) fn emit_opencode_message_events( let mut rows = statement.query([])?; let mut reconstructed_usage = HashMap::::new(); let mut session_models = HashMap::::new(); + let mut pending_user_messages = HashMap::::new(); while let Some(row) = rows.next()? { let session_id: String = row.get(1)?; if !ctx.reconstructed_session_ids.contains(&session_id) { @@ -776,7 +815,11 @@ pub(crate) fn emit_opencode_message_events( session_models.insert(session_id.clone(), model); } let usage = opencode_message_usage_counts(&value); + let role = value.get("role").and_then(Value::as_str); if usage.computed_total() == 0 { + if role == Some("user") { + *pending_user_messages.entry(session_id).or_default() += 1; + } ctx.scan.diagnostics.skipped_zero_events += 1; continue; } @@ -785,6 +828,10 @@ pub(crate) fn emit_opencode_message_events( ctx.scan.diagnostics.model_fallbacks += 1; continue; }; + let pending_users = pending_user_messages.remove(&session_id).unwrap_or(0); + let user_messages = pending_users.saturating_add(u64::from(role == Some("user"))); + let assistant_messages = u64::from(role != Some("user") && role != Some("developer")); + let developer_messages = u64::from(role == Some("developer")); let started_at = value .pointer("/time/created") .and_then(value_as_u64) @@ -817,7 +864,11 @@ pub(crate) fn emit_opencode_message_events( duration_seconds, model: Some(model), usage, - runtime: None, + runtime: message_count_runtime( + user_messages, + assistant_messages, + developer_messages, + ), session_raw: session_id.clone(), project, event_kind: "opencode_message_usage", diff --git a/crates/statsai-core/src/tests/types.rs b/crates/statsai-core/src/tests/types.rs index d5103dd..5386219 100644 --- a/crates/statsai-core/src/tests/types.rs +++ b/crates/statsai-core/src/tests/types.rs @@ -1,5 +1,6 @@ use super::support::*; use super::*; +use chrono::{DateTime, Utc}; use std::path::Path; #[test] fn source_ids_are_stable_for_same_input() { @@ -58,6 +59,82 @@ fn schema_types_serialize() { let schema = schemars::schema_for!(UsageSummary); let json = serde_json::to_value(schema).expect("summary schema should serialize"); assert!(json.get("title").is_some()); + + let schema = schemars::schema_for!(SessionRollupV1); + let json = serde_json::to_value(schema).expect("session rollup schema should serialize"); + assert!(json.get("title").is_some()); +} + +#[test] +fn session_rollup_uses_snake_case_contract() { + let started = DateTime::parse_from_rfc3339("2026-05-01T00:00:00Z") + .expect("start") + .with_timezone(&Utc); + let rollup = SessionRollupV1 { + schema_version: SESSION_ROLLUP_SCHEMA_VERSION.to_string(), + session_id: "session_abc".to_string(), + device_id: "device".to_string(), + provider: "codex".to_string(), + source_id: SourceId("src".to_string()), + provider_account_id: None, + started_at: started, + ended_at: started, + duration_seconds: Some(0), + usage: UsageCounts::default(), + requests: 1, + cost: CostInfo { + currency: "USD".to_string(), + estimated_api_equivalent_usd: None, + provider_reported_usd: None, + estimated_api_equivalent_micro_usd: Some(10), + provider_reported_micro_usd: None, + pricing_source: None, + pricing_version: None, + confidence: Confidence::Medium, + }, + models: Vec::new(), + primary_model: Some("gpt-5".to_string()), + total_messages: Some(2), + user_messages: Some(1), + assistant_messages: Some(1), + developer_messages: None, + project: None, + title: Some("Fix parser".to_string()), + title_source: Some(SessionTitleSource::TaskSpan), + updated_at: started, + }; + let json = serde_json::to_value(&rollup).expect("serialize"); + for field in [ + "schema_version", + "session_id", + "device_id", + "provider", + "source_id", + "provider_account_id", + "started_at", + "ended_at", + "duration_seconds", + "usage", + "requests", + "cost", + "models", + "primary_model", + "total_messages", + "user_messages", + "assistant_messages", + "developer_messages", + "project", + "title", + "title_source", + "updated_at", + ] { + assert!(json.get(field).is_some(), "missing {field}"); + } + assert_eq!(json["schema_version"], "session_rollup.v1"); + assert_eq!(json["title_source"], "task_span"); + assert_eq!(json["requests"], 1); + let restored: SessionRollupV1 = serde_json::from_value(json).expect("deserialize"); + assert_eq!(restored, rollup); } #[test] diff --git a/crates/statsai-core/src/types/mod.rs b/crates/statsai-core/src/types/mod.rs index 952a554..328f5fb 100644 --- a/crates/statsai-core/src/types/mod.rs +++ b/crates/statsai-core/src/types/mod.rs @@ -1,6 +1,7 @@ mod account; mod activity; mod event; +mod session; mod source; mod subscription; mod summary; @@ -9,6 +10,7 @@ mod sync; pub use account::*; pub use activity::*; pub use event::*; +pub use session::*; pub use source::*; pub use subscription::*; pub use summary::*; diff --git a/crates/statsai-core/src/types/session.rs b/crates/statsai-core/src/types/session.rs new file mode 100644 index 0000000..1b8b7d2 --- /dev/null +++ b/crates/statsai-core/src/types/session.rs @@ -0,0 +1,48 @@ +use super::event::{CostInfo, ProjectInfo, UsageCounts}; +use super::summary::SummaryModelUsage; +use crate::ids::{ProviderAccountId, SourceId}; +use chrono::{DateTime, Utc}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +pub const SESSION_ROLLUP_SCHEMA_VERSION: &str = "session_rollup.v1"; + +/// Where a session title was resolved from. Event titles win over task spans, +/// which win over archived conversations. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum SessionTitleSource { + Event, + TaskSpan, + Archive, +} + +/// One local session, aggregated from the usage events that share its hashed id. +/// +/// `session_id` is the hashed `session_…` identity already stored on each event. +/// Raw provider session ids never appear on this record. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct SessionRollupV1 { + pub schema_version: String, + pub session_id: String, + pub device_id: String, + pub provider: String, + pub source_id: SourceId, + pub provider_account_id: Option, + pub started_at: DateTime, + pub ended_at: DateTime, + pub duration_seconds: Option, + pub usage: UsageCounts, + pub requests: u64, + pub cost: CostInfo, + pub models: Vec, + pub primary_model: Option, + pub total_messages: Option, + pub user_messages: Option, + pub assistant_messages: Option, + pub developer_messages: Option, + pub project: Option, + pub title: Option, + pub title_source: Option, + pub updated_at: DateTime, +} diff --git a/crates/statsai-store/src/account_plan/observations.rs b/crates/statsai-store/src/account_plan/observations.rs index f177570..10dd3ea 100644 --- a/crates/statsai-store/src/account_plan/observations.rs +++ b/crates/statsai-store/src/account_plan/observations.rs @@ -571,15 +571,16 @@ impl Store { .collect::>(); self.apply_conversation_account_bindings(source_id, &mut events)?; let mut changed = 0u64; - let mut dirty_keys = BTreeSet::new(); + let mut dirty = crate::events::EventWriteDirty::default(); for (event, previous_account) in events.iter().zip(previous_accounts) { if event.provider_account_id == previous_account { continue; } - dirty_keys.extend(self.update_event_payload(event)?); + dirty.extend(self.update_event_payload(event)?); changed += 1; } - self.refresh_sync_rollups_for_keys(&dirty_keys)?; + self.refresh_sync_rollups_for_keys(&dirty.buckets)?; + self.refresh_session_rollups_for_keys(&dirty.sessions)?; Ok(changed) } diff --git a/crates/statsai-store/src/archive/write.rs b/crates/statsai-store/src/archive/write.rs index 873d591..e16659c 100644 --- a/crates/statsai-store/src/archive/write.rs +++ b/crates/statsai-store/src/archive/write.rs @@ -201,6 +201,11 @@ impl Store { ], )?; } + let raw_session_ids = conversations + .iter() + .map(|conversation| conversation.native_conversation_id.clone()) + .collect::>(); + self.refresh_session_rollups_for_raw_ids(&raw_session_ids)?; Ok(result) } diff --git a/crates/statsai-store/src/events.rs b/crates/statsai-store/src/events.rs index 77680d7..e31c566 100644 --- a/crates/statsai-store/src/events.rs +++ b/crates/statsai-store/src/events.rs @@ -1,11 +1,35 @@ use super::*; use crate::dedupe::*; +#[derive(Debug, Default)] +pub(crate) struct EventWriteDirty { + pub(crate) buckets: BTreeSet, + pub(crate) sessions: BTreeSet, +} + +impl EventWriteDirty { + fn from_event(event: &UsageEvent) -> Self { + let mut dirty = Self::default(); + dirty.observe(event); + dirty + } + + fn observe(&mut self, event: &UsageEvent) { + self.buckets.insert(sync_rollup_bucket_key(event)); + self.sessions.insert(event.session.session_id.clone()); + } + + pub(crate) fn extend(&mut self, other: Self) { + self.buckets.extend(other.buckets); + self.sessions.extend(other.sessions); + } +} + #[derive(Debug)] struct EventInsertOutcome { inserted: bool, canonical_event_id: EventId, - dirty_keys: BTreeSet, + dirty: EventWriteDirty, } impl Store { @@ -16,8 +40,8 @@ impl Store { let existing = self.event_by_id(&existing_id)?; let refreshed = refreshed_duplicate_event(existing.as_ref(), &event, existing_id.as_str()); - let dirty_keys = self.update_event_payload(&refreshed)?; - self.refresh_sync_rollups_for_keys(&dirty_keys)?; + let dirty = self.update_event_payload(&refreshed)?; + self.refresh_event_rollups(&dirty)?; return Ok(false); } @@ -45,10 +69,10 @@ impl Store { let existing = self.event_by_id(&event.event_id.0)?; let refreshed = refreshed_duplicate_event(existing.as_ref(), &event, event.event_id.0.as_str()); - let dirty_keys = self.update_event_payload(&refreshed)?; - self.refresh_sync_rollups_for_keys(&dirty_keys)?; + let dirty = self.update_event_payload(&refreshed)?; + self.refresh_event_rollups(&dirty)?; } else { - self.refresh_sync_rollups_for_keys(&BTreeSet::from([sync_rollup_bucket_key(&event)]))?; + self.refresh_event_rollups(&EventWriteDirty::from_event(&event))?; } Ok(changed > 0) } @@ -75,7 +99,7 @@ impl Store { let mut conflict_map = self.batch_load_conflicts(&conflict_keys)?; let mut inserted = 0u64; let mut canonical_event_ids = HashMap::with_capacity(events.len()); - let mut dirty_keys = BTreeSet::new(); + let mut dirty = EventWriteDirty::default(); for (index, event) in events.iter().enumerate() { let incoming_event_id = event.event_id.clone(); let matched_event = @@ -99,7 +123,7 @@ impl Store { if let Some((existing_id, existing)) = matched_event { let refreshed = refreshed_duplicate_event(Some(&existing), event, existing_id.as_str()); - dirty_keys.extend(self.update_event_payload(&refreshed)?); + dirty.extend(self.update_event_payload(&refreshed)?); canonical_event_ids.insert(incoming_event_id, EventId(existing_id.clone())); let candidates = conflict_map .entry(conflict_keys[index].clone()) @@ -123,7 +147,7 @@ impl Store { inserted += 1; } canonical_event_ids.insert(incoming_event_id, outcome.canonical_event_id.clone()); - dirty_keys.extend(outcome.dirty_keys); + dirty.extend(outcome.dirty); conflict_map .entry(conflict_keys[index].clone()) .or_default() @@ -132,7 +156,7 @@ impl Store { event: event.clone(), }); } - self.refresh_sync_rollups_for_keys(&dirty_keys)?; + self.refresh_event_rollups(&dirty)?; Ok(EventInsertBatchResult { inserted, canonical_event_ids, @@ -140,20 +164,20 @@ impl Store { }) } - pub(crate) fn update_event_payload( - &self, - event: &UsageEvent, - ) -> Result> { - let existing_bucket = self - .event_by_id(&event.event_id.0)? - .map(|existing| sync_rollup_bucket_key(&existing)); - let bucket = self.update_event_cost_payload(event)?; - let mut dirty_keys = BTreeSet::new(); - if let Some(existing_bucket) = existing_bucket { - dirty_keys.insert(existing_bucket); + fn refresh_event_rollups(&self, dirty: &EventWriteDirty) -> Result<()> { + self.refresh_sync_rollups_for_keys(&dirty.buckets)?; + self.refresh_session_rollups_for_keys(&dirty.sessions)?; + Ok(()) + } + + pub(crate) fn update_event_payload(&self, event: &UsageEvent) -> Result { + let existing = self.event_by_id(&event.event_id.0)?; + self.update_event_cost_payload(event)?; + let mut dirty = EventWriteDirty::from_event(event); + if let Some(existing) = existing.as_ref() { + dirty.observe(existing); } - dirty_keys.insert(bucket); - Ok(dirty_keys) + Ok(dirty) } /// Updates a persisted event's payload without re-reading it. @@ -224,13 +248,13 @@ impl Store { return Ok(EventInsertOutcome { inserted: false, canonical_event_id: event.event_id.clone(), - dirty_keys: self.update_event_payload(&refreshed)?, + dirty: self.update_event_payload(&refreshed)?, }); } Ok(EventInsertOutcome { inserted: true, canonical_event_id: event.event_id.clone(), - dirty_keys: BTreeSet::from([sync_rollup_bucket_key(event)]), + dirty: EventWriteDirty::from_event(event), }) } @@ -569,12 +593,12 @@ impl Store { } self.with_immediate_transaction(|| { let mut changed = 0u64; - let mut dirty_keys = BTreeSet::new(); + let mut dirty = EventWriteDirty::default(); for event in events { - dirty_keys.extend(self.update_event_payload(event)?); + dirty.extend(self.update_event_payload(event)?); changed += 1; } - self.refresh_sync_rollups_for_keys(&dirty_keys)?; + self.refresh_event_rollups(&dirty)?; Ok(changed) }) } @@ -605,6 +629,7 @@ impl Store { )? as u64; } self.delete_sync_rollups_for_sources_in_tx(source_ids)?; + self.delete_session_rollups_for_sources_in_tx(source_ids)?; Ok(impact) }) } @@ -620,7 +645,7 @@ impl Store { self.with_immediate_transaction(|| { let mut impact = EventDeletionImpact::default(); - let mut dirty_keys = BTreeSet::new(); + let mut dirty = EventWriteDirty::default(); // Both statements filter on the expression `usage_events_source_file_idx` // indexes, so they read an index range rather than the whole table. @@ -643,7 +668,7 @@ impl Store { for payload in payloads { let event: UsageEvent = serde_json::from_str(&payload)?; - dirty_keys.insert(sync_rollup_bucket_key(&event)); + dirty.observe(&event); impact.deleted_event_ids.push(event.event_id); } @@ -653,7 +678,7 @@ impl Store { as u64; } - self.refresh_sync_rollups_for_keys(&dirty_keys)?; + self.refresh_event_rollups(&dirty)?; Ok(impact) }) } diff --git a/crates/statsai-store/src/lib.rs b/crates/statsai-store/src/lib.rs index f34c59b..63903e2 100644 --- a/crates/statsai-store/src/lib.rs +++ b/crates/statsai-store/src/lib.rs @@ -80,6 +80,7 @@ pub use pricing::{ apply_current_estimated_pricing, RepricingReport, APPLIED_PRICING_CATALOG_VERSION_KEY, APPLIED_PRICING_RULESET_VERSION_KEY, }; +pub use rollups::{SessionFilter, SessionSort, SessionStats}; pub use snapshot::{ clone_database_to, database_applied_pricing_ruleset_version, database_schema_version, DatabaseClone, @@ -142,6 +143,10 @@ pub(crate) const EVENT_SOURCE_FILE_HASH_SQL: &str = "CASE WHEN json_valid(payloa THEN json_extract(payload, '$.parse_evidence.source_file_path_hash') END"; pub(crate) const EVENT_CONVERSATION_HASH_SQL: &str = "CASE WHEN json_valid(payload) \ THEN json_extract(payload, '$.session.local_session_id_hash') END"; +/// Hashed `session_…` id. Indexed so a session refresh reads one conversation +/// instead of scanning every usage payload. +pub(crate) const EVENT_SESSION_ID_SQL: &str = "CASE WHEN json_valid(payload) \ + THEN json_extract(payload, '$.session.session_id') END"; /// The one field plan evidence needs out of a quota observation payload. pub(crate) const QUOTA_PLAN_TYPE_SQL: &str = "CASE WHEN json_valid(payload) \ THEN json_extract(payload, '$.status.plan_type') END"; @@ -355,6 +360,7 @@ impl Store { let store = Self { conn }; store.migrate()?; store.configure_connection()?; + store.backfill_session_rollups_if_needed()?; store.conn.execute_batch("PRAGMA optimize=0x10002;")?; Ok(store) } @@ -436,6 +442,7 @@ impl Store { store.conn.busy_timeout(SQLITE_BUSY_TIMEOUT)?; store.migrate()?; store.configure_connection()?; + store.backfill_session_rollups_if_needed()?; store.conn.execute_batch("PRAGMA optimize=0x10002;")?; Ok(store) } diff --git a/crates/statsai-store/src/migrations/mod.rs b/crates/statsai-store/src/migrations/mod.rs index d4887d6..3c31083 100644 --- a/crates/statsai-store/src/migrations/mod.rs +++ b/crates/statsai-store/src/migrations/mod.rs @@ -8,7 +8,7 @@ mod v2; pub(crate) use v1::*; pub(crate) use v2::*; -pub const CURRENT_SCHEMA_VERSION: i64 = 25; +pub const CURRENT_SCHEMA_VERSION: i64 = 26; pub fn migrate(conn: &Connection) -> Result<()> { if let Some(current) = existing_schema_version(conn)? { @@ -150,6 +150,7 @@ fn apply_migration(conn: &Connection, version: i64) -> Result<()> { 23 => apply_migration_023(conn), 24 => apply_migration_024(conn), 25 => apply_migration_025(conn), + 26 => apply_migration_026(conn), _ => bail!("unsupported schema migration version {version}"), } } @@ -442,21 +443,27 @@ mod tests { version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL ); - INSERT INTO schema_migrations (version, applied_at) - VALUES (26, '2026-08-23T00:00:00Z'); "#, ) + .expect("prepare future schema marker"); + let future = CURRENT_SCHEMA_VERSION + 1; + conn.execute( + "INSERT INTO schema_migrations (version, applied_at) VALUES (?1, '2026-08-23T00:00:00Z')", + [future], + ) .expect("create future schema marker"); - let error = migrate(&conn).expect_err("schema 26 must be rejected by schema 25 binary"); + let error = migrate(&conn).expect_err("newer schema must be rejected"); assert_eq!( error.to_string(), - "database schema version 26 is newer than this StatsAI binary supports (25); upgrade StatsAI or use a compatible database" + format!( + "database schema version {future} is newer than this StatsAI binary supports ({CURRENT_SCHEMA_VERSION}); upgrade StatsAI or use a compatible database" + ) ); assert_eq!( current_schema_version(&conn).expect("read unchanged version"), - 26 + future ); } diff --git a/crates/statsai-store/src/migrations/v2.rs b/crates/statsai-store/src/migrations/v2.rs index 8f87d33..c41d923 100644 --- a/crates/statsai-store/src/migrations/v2.rs +++ b/crates/statsai-store/src/migrations/v2.rs @@ -1,5 +1,8 @@ use super::*; -use crate::{EVENT_CONVERSATION_HASH_SQL, EVENT_SOURCE_FILE_HASH_SQL, QUOTA_PLAN_TYPE_SQL}; +use crate::{ + EVENT_CONVERSATION_HASH_SQL, EVENT_SESSION_ID_SQL, EVENT_SOURCE_FILE_HASH_SQL, + QUOTA_PLAN_TYPE_SQL, +}; use rusqlite::params; use statsai_core::{account_plan_observation_id, AccountPlanObservationV1}; @@ -548,3 +551,30 @@ pub(crate) fn apply_migration_025(conn: &Connection) -> Result<()> { )?; Ok(()) } + +pub(crate) fn apply_migration_026(conn: &Connection) -> Result<()> { + conn.execute_batch(&format!( + r#" + CREATE TABLE IF NOT EXISTS session_rollups ( + session_id TEXT PRIMARY KEY, + provider TEXT NOT NULL, + source_id TEXT NOT NULL, + provider_account_id TEXT, + day_key TEXT NOT NULL, + started_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + total_tokens INTEGER NOT NULL, + payload_hash TEXT NOT NULL, + dirty INTEGER NOT NULL DEFAULT 1, + payload TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS session_rollups_started_idx + ON session_rollups (started_at); + CREATE INDEX IF NOT EXISTS session_rollups_provider_day_idx + ON session_rollups (provider, day_key); + CREATE INDEX IF NOT EXISTS usage_events_session_idx + ON usage_events ({EVENT_SESSION_ID_SQL}); + "#, + ))?; + Ok(()) +} diff --git a/crates/statsai-store/src/pricing/mod.rs b/crates/statsai-store/src/pricing/mod.rs index 01225b3..d0b0abe 100644 --- a/crates/statsai-store/src/pricing/mod.rs +++ b/crates/statsai-store/src/pricing/mod.rs @@ -130,10 +130,12 @@ impl Store { let mut report = RepricingReport::default(); let mut dirty_keys = BTreeSet::new(); + let mut session_ids = BTreeSet::new(); - self.reprice_events_in_tx(&mut report, &mut dirty_keys)?; + self.reprice_events_in_tx(&mut report, &mut dirty_keys, &mut session_ids)?; self.reprice_summaries_in_tx(&mut report)?; report.refreshed_rollups = self.refresh_sync_rollups_for_keys_counted(&dirty_keys)?; + self.refresh_session_rollups_for_keys(&session_ids)?; self.reprice_task_spans_in_tx(&mut report)?; self.set_metadata_value( @@ -185,6 +187,7 @@ impl Store { &self, report: &mut RepricingReport, dirty_keys: &mut BTreeSet, + session_ids: &mut BTreeSet, ) -> Result<()> { let mut after: Option = None; loop { @@ -199,6 +202,7 @@ impl Store { maybe_fail_after_event_writes(report.changed_events)?; if let Some(updated) = reprice_event(&event) { dirty_keys.insert(self.update_event_cost_payload(&updated)?); + session_ids.insert(updated.session.session_id.clone()); report.changed_events += 1; } } diff --git a/crates/statsai-store/src/pricing/tests/repricing.rs b/crates/statsai-store/src/pricing/tests/repricing.rs index 7dd18e1..a4c8094 100644 --- a/crates/statsai-store/src/pricing/tests/repricing.rs +++ b/crates/statsai-store/src/pricing/tests/repricing.rs @@ -384,7 +384,7 @@ fn daily_rollups_are_unused_by_report_and_sync_paths() { assert_eq!(events[0].cost, expected_review_cost(started_at)); // Pin: bump this when CURRENT_SCHEMA_VERSION changes, after confirming // the daily_rollups table is still unused by report/sync/snapshot. - assert_eq!(CURRENT_SCHEMA_VERSION, 25); + assert_eq!(CURRENT_SCHEMA_VERSION, 26); } #[test] diff --git a/crates/statsai-store/src/rollups/mod.rs b/crates/statsai-store/src/rollups/mod.rs index 7dc846f..5407b84 100644 --- a/crates/statsai-store/src/rollups/mod.rs +++ b/crates/statsai-store/src/rollups/mod.rs @@ -1,8 +1,11 @@ use super::*; mod build; +mod sessions; mod summaries; +pub use sessions::{SessionFilter, SessionSort, SessionStats}; + pub(crate) use build::*; pub(crate) use summaries::*; diff --git a/crates/statsai-store/src/rollups/sessions.rs b/crates/statsai-store/src/rollups/sessions.rs new file mode 100644 index 0000000..1db5c93 --- /dev/null +++ b/crates/statsai-store/src/rollups/sessions.rs @@ -0,0 +1,922 @@ +use crate::{ + safe_u64_to_i64, sqlite_in_clause_placeholders, sqlite_string_params, Store, + EVENT_CONVERSATION_HASH_SQL, EVENT_SESSION_ID_SQL, +}; +use anyhow::Result; +use chrono::{DateTime, Utc}; +use rusqlite::{params, OptionalExtension}; +use statsai_core::{ + hash_text, project_has_stable_identity, task_title_is_generic, Confidence, CostAccumulator, + CostInfo, ModelInfo, SessionRollupV1, SessionTitleSource, SummaryModelUsage, UsageCounts, + UsageEvent, SESSION_ROLLUP_SCHEMA_VERSION, +}; +use std::collections::{BTreeMap, BTreeSet}; + +const SESSION_ROLLUPS_BACKFILL_KEY: &str = "session_rollups_backfilled_v1"; +const UNASSIGNED_ACCOUNT: &str = "unassigned"; + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SessionFilter { + pub provider: Option, + pub project_key: Option, + pub account: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum SessionSort { + #[default] + Started, + Tokens, + Cost, + Duration, + Messages, +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +pub struct SessionStats { + pub sessions: u64, + pub total_tokens: u64, + pub total_cost_micro_usd: i64, + pub avg_tokens: Option, + pub median_tokens: Option, + pub avg_messages: Option, + pub median_duration_seconds: Option, + pub total_messages: u64, + pub top_model: Option, +} + +impl Store { + pub(crate) fn backfill_session_rollups_if_needed(&self) -> Result<()> { + if self + .metadata_value(SESSION_ROLLUPS_BACKFILL_KEY)? + .as_deref() + == Some("1") + { + return Ok(()); + } + self.rebuild_session_rollups()?; + self.set_metadata_value(SESSION_ROLLUPS_BACKFILL_KEY, "1")?; + Ok(()) + } + + pub fn rebuild_session_rollups(&self) -> Result { + let events = self.events()?; + let mut grouped: BTreeMap> = BTreeMap::new(); + for event in events { + grouped + .entry(event.session.session_id.clone()) + .or_default() + .push(event); + } + self.with_immediate_transaction(|| { + self.conn.execute("DELETE FROM session_rollups", [])?; + let hashes = grouped + .values() + .flatten() + .filter_map(|event| event.session.local_session_id_hash.clone()) + .collect::>(); + let titles = self.session_titles_for_hashes(&hashes)?; + let updated_at = Utc::now(); + for events in grouped.values() { + let rollup = build_session_rollup(events, &titles, updated_at); + let payload_hash = session_rollup_content_hash(&rollup)?; + self.write_session_rollup(&rollup, &payload_hash, 1)?; + } + Ok(grouped.len() as u64) + }) + } + + pub(crate) fn refresh_session_rollups_for_keys( + &self, + session_ids: &BTreeSet, + ) -> Result<()> { + if session_ids.is_empty() { + return Ok(()); + } + let mut present = BTreeMap::new(); + let mut hashes = Vec::new(); + for session_id in session_ids { + let events = self.session_events(session_id)?; + if events.is_empty() { + self.conn.execute( + "DELETE FROM session_rollups WHERE session_id = ?1", + params![session_id], + )?; + continue; + } + hashes.extend( + events + .iter() + .filter_map(|event| event.session.local_session_id_hash.clone()), + ); + present.insert(session_id.clone(), events); + } + // Title resolution scans task spans and archives once per batch. Doing + // it per session turns a scan into one full pass for every conversation. + let titles = self.session_titles_for_hashes(&hashes)?; + for (session_id, events) in present { + let rollup = build_session_rollup(&events, &titles, Utc::now()); + let payload_hash = session_rollup_content_hash(&rollup)?; + let existing: Option = self + .conn + .query_row( + "SELECT payload_hash FROM session_rollups WHERE session_id = ?1", + params![session_id], + |row| row.get(0), + ) + .optional()?; + if existing.as_deref() == Some(payload_hash.as_str()) { + continue; + } + self.write_session_rollup(&rollup, &payload_hash, 1)?; + } + Ok(()) + } + + pub(crate) fn refresh_session_rollups_for_raw_ids(&self, raw_ids: &[String]) -> Result<()> { + let hashes = raw_ids + .iter() + .map(|raw_id| raw_id.trim()) + .filter(|raw_id| !raw_id.is_empty()) + .map(hash_text) + .collect::>(); + let session_ids = self.session_ids_for_local_hashes(&hashes)?; + self.refresh_session_rollups_for_keys(&session_ids) + } + + fn write_session_rollup( + &self, + rollup: &SessionRollupV1, + payload_hash: &str, + dirty: i64, + ) -> Result<()> { + let payload = serde_json::to_string(rollup)?; + let day_key = rollup + .started_at + .date_naive() + .format("%Y-%m-%d") + .to_string(); + self.conn.execute( + r#" + INSERT INTO session_rollups ( + session_id, provider, source_id, provider_account_id, day_key, + started_at, updated_at, total_tokens, payload_hash, dirty, payload + ) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) + ON CONFLICT(session_id) DO UPDATE SET + provider = excluded.provider, + source_id = excluded.source_id, + provider_account_id = excluded.provider_account_id, + day_key = excluded.day_key, + started_at = excluded.started_at, + updated_at = excluded.updated_at, + total_tokens = excluded.total_tokens, + payload_hash = excluded.payload_hash, + dirty = excluded.dirty, + payload = excluded.payload + "#, + params![ + &rollup.session_id, + &rollup.provider, + &rollup.source_id.0, + rollup.provider_account_id.as_ref().map(|id| id.0.as_str()), + &day_key, + rollup.started_at.to_rfc3339(), + rollup.updated_at.to_rfc3339(), + safe_u64_to_i64(rollup.usage.computed_total()), + payload_hash, + dirty, + &payload, + ], + )?; + Ok(()) + } + + pub fn dirty_session_rollups(&self) -> Result> { + self.session_rollups_by_sql( + "SELECT payload FROM session_rollups WHERE dirty = 1 ORDER BY started_at, session_id", + ) + } + + pub fn mark_session_rollups_synced(&self, session_ids: &[String]) -> Result<()> { + if session_ids.is_empty() { + return Ok(()); + } + self.with_immediate_transaction(|| { + self.mark_session_rollups_synced_in_transaction(session_ids) + }) + } + + pub(crate) fn mark_session_rollups_synced_in_transaction( + &self, + session_ids: &[String], + ) -> Result<()> { + for session_id in session_ids { + self.conn.execute( + "UPDATE session_rollups SET dirty = 0 WHERE session_id = ?1", + params![session_id], + )?; + } + Ok(()) + } + + pub fn mark_all_session_rollups_dirty(&self) -> Result { + let updated = self.conn.execute( + "UPDATE session_rollups SET dirty = 1, updated_at = ?1", + params![Utc::now().to_rfc3339()], + )? as u64; + Ok(updated) + } + + pub(crate) fn delete_session_rollups_for_sources_in_tx( + &self, + source_ids: &[statsai_core::SourceId], + ) -> Result { + let mut deleted = 0u64; + for source_id in source_ids { + deleted += self.conn.execute( + "DELETE FROM session_rollups WHERE source_id = ?1", + params![&source_id.0], + )? as u64; + } + Ok(deleted) + } + + pub fn session_rollups_in_period( + &self, + since: Option>, + until: DateTime, + filter: &SessionFilter, + sort: SessionSort, + limit: usize, + offset: usize, + ) -> Result> { + let mut rollups = self.filtered_session_rollups(since, until, filter)?; + sort_sessions(&mut rollups, sort); + Ok(rollups.into_iter().skip(offset).take(limit).collect()) + } + + pub fn session_stats_in_period( + &self, + since: Option>, + until: DateTime, + filter: &SessionFilter, + ) -> Result { + let rollups = self.filtered_session_rollups(since, until, filter)?; + Ok(session_stats(&rollups)) + } + + fn filtered_session_rollups( + &self, + since: Option>, + until: DateTime, + filter: &SessionFilter, + ) -> Result> { + let since = since.map(|value| value.to_rfc3339()); + let until = until.to_rfc3339(); + let provider = filter.provider.as_deref(); + let account = filter.account.as_deref(); + let mut statement = self.conn.prepare( + r#" + SELECT payload FROM session_rollups + WHERE started_at <= ?1 + AND (?2 IS NULL OR started_at >= ?2) + AND (?3 IS NULL OR provider = ?3) + AND ( + ?4 IS NULL + OR (?4 = ?5 AND provider_account_id IS NULL) + OR provider_account_id = ?4 + ) + ORDER BY started_at, session_id + "#, + )?; + let rows = statement.query_map( + params![until, since, provider, account, UNASSIGNED_ACCOUNT], + |row| row.get::<_, String>(0), + )?; + let mut rollups = Vec::new(); + for row in rows { + let rollup: SessionRollupV1 = serde_json::from_str(&row?)?; + if session_matches_project(&rollup, filter.project_key.as_deref()) { + rollups.push(rollup); + } + } + Ok(rollups) + } + + fn session_rollups_by_sql(&self, sql: &str) -> Result> { + let mut statement = self.conn.prepare(sql)?; + let rows = statement.query_map([], |row| row.get::<_, String>(0))?; + let mut rollups = Vec::new(); + for row in rows { + rollups.push(serde_json::from_str(&row?)?); + } + Ok(rollups) + } + + fn session_events(&self, session_id: &str) -> Result> { + let sql = format!( + "SELECT payload FROM usage_events WHERE {EVENT_SESSION_ID_SQL} = ?1 ORDER BY started_at, event_id" + ); + let mut statement = self.conn.prepare(&sql)?; + let rows = statement.query_map(params![session_id], |row| row.get::<_, String>(0))?; + let mut events = Vec::new(); + for row in rows { + events.push(serde_json::from_str(&row?)?); + } + Ok(events) + } + + fn session_ids_for_local_hashes(&self, hashes: &[String]) -> Result> { + let mut session_ids = BTreeSet::new(); + if hashes.is_empty() { + return Ok(session_ids); + } + for chunk in hashes.chunks(500) { + let placeholders = sqlite_in_clause_placeholders(chunk.len()); + let sql = format!( + "SELECT DISTINCT json_extract(payload, '$.session.session_id') + FROM usage_events + WHERE {EVENT_CONVERSATION_HASH_SQL} IN ({placeholders}) + AND json_extract(payload, '$.session.session_id') IS NOT NULL" + ); + let mut statement = self.conn.prepare(&sql)?; + let params = sqlite_string_params(chunk); + let rows = statement.query_map(params.as_slice(), |row| row.get::<_, String>(0))?; + for row in rows { + session_ids.insert(row?); + } + } + Ok(session_ids) + } + + /// Titles keyed by `local_session_id_hash`. Task spans replace archive titles. + /// Generic and secret-looking titles are dropped. + pub(crate) fn session_titles_for_hashes( + &self, + hashes: &[String], + ) -> Result> { + let wanted = hashes.iter().map(String::as_str).collect::>(); + if wanted.is_empty() { + return Ok(BTreeMap::new()); + } + let mut titles = BTreeMap::new(); + let mut archive = self.conn.prepare( + "SELECT native_conversation_id, title + FROM archive_conversations + WHERE title IS NOT NULL + ORDER BY updated_at, conversation_id", + )?; + let archive_rows = archive.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)) + })?; + for row in archive_rows { + let (native_id, title) = row?; + remember_hashed_title( + &mut titles, + &wanted, + &native_id, + title.as_deref(), + SessionTitleSource::Archive, + ); + } + drop(archive); + + let mut spans = self.conn.prepare( + "SELECT json_extract(payload, '$.session_id'), + json_extract(payload, '$.title'), + json_extract(payload, '$.is_meta') + FROM task_spans + ORDER BY started_at, span_id", + )?; + let span_rows = spans.query_map([], |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + )) + })?; + for row in span_rows { + let (session_id, title, is_meta) = row?; + if is_meta.unwrap_or(0) != 0 { + continue; + } + let Some(session_id) = session_id.filter(|value| !value.trim().is_empty()) else { + continue; + }; + remember_hashed_title( + &mut titles, + &wanted, + &session_id, + title.as_deref(), + SessionTitleSource::TaskSpan, + ); + } + Ok(titles) + } + + pub(crate) fn task_span_raw_session_ids_for_bucket( + &self, + project_bucket: &str, + ) -> Result> { + let mut statement = self.conn.prepare( + "SELECT json_extract(payload, '$.session_id') + FROM task_spans + WHERE project_bucket = ?1", + )?; + let rows = statement.query_map(params![project_bucket], |row| { + row.get::<_, Option>(0) + })?; + let mut ids = BTreeSet::new(); + for row in rows { + if let Some(session_id) = row?.filter(|value| !value.trim().is_empty()) { + ids.insert(session_id); + } + } + Ok(ids) + } +} + +pub(crate) fn build_session_rollup( + events: &[UsageEvent], + titles: &BTreeMap, + updated_at: DateTime, +) -> SessionRollupV1 { + let mut ordered = events.to_vec(); + ordered.sort_by(|left, right| { + left.session + .started_at + .cmp(&right.session.started_at) + .then(left.created_at.cmp(&right.created_at)) + .then(left.event_id.0.cmp(&right.event_id.0)) + }); + let events = ordered.as_slice(); + let oldest = events.first().expect("session rollup requires events"); + let newest = events.last().unwrap_or(oldest); + let mut totals = UsageTotals::default(); + let mut estimated_cost = CostAccumulator::default(); + let mut provider_reported_cost = CostAccumulator::default(); + let mut has_provider_reported = false; + let mut model_buckets: BTreeMap = BTreeMap::new(); + let mut total_messages = 0u64; + let mut user_messages = 0u64; + let mut assistant_messages = 0u64; + let mut developer_messages = 0u64; + let mut saw_messages = false; + + for event in events { + totals.add(&event.usage); + let requests = event.usage.requests.unwrap_or(1); + totals.requests = totals.requests.saturating_add(requests); + estimated_cost.add_estimated(&event.cost); + if event.cost.provider_reported_micro_usd.is_some() + || event.cost.provider_reported_usd.is_some() + { + provider_reported_cost.add_values( + event.cost.provider_reported_micro_usd, + event.cost.provider_reported_usd, + ); + has_provider_reported = true; + } + if let Some(runtime) = event.runtime.as_ref() { + let derived_total = runtime.total_messages.or_else(|| { + let derived = runtime + .user_messages + .unwrap_or(0) + .saturating_add(runtime.assistant_messages.unwrap_or(0)) + .saturating_add(runtime.developer_messages.unwrap_or(0)); + (derived > 0).then_some(derived) + }); + if derived_total.is_some() + || runtime.user_messages.is_some() + || runtime.assistant_messages.is_some() + || runtime.developer_messages.is_some() + { + saw_messages = true; + } + total_messages = total_messages.saturating_add(derived_total.unwrap_or(0)); + user_messages = user_messages.saturating_add(runtime.user_messages.unwrap_or(0)); + assistant_messages = + assistant_messages.saturating_add(runtime.assistant_messages.unwrap_or(0)); + developer_messages = + developer_messages.saturating_add(runtime.developer_messages.unwrap_or(0)); + } + + let model = event.model.clone().unwrap_or_default(); + let entry = model_buckets + .entry(session_model_key(&model)) + .or_insert_with(|| (model, ModelTotals::default())); + entry.1.add(&event.usage, requests, &event.cost); + } + + let started_at = events + .iter() + .map(|event| event.session.started_at) + .min() + .unwrap_or(oldest.session.started_at); + let ended_at = events + .iter() + .map(|event| event.session.ended_at.unwrap_or(event.created_at)) + .max() + .unwrap_or(newest.created_at); + let duration_seconds = ended_at + .signed_duration_since(started_at) + .num_seconds() + .try_into() + .ok(); + let provider_account_id = events + .iter() + .rev() + .find_map(|event| event.provider_account_id.clone()); + let project = newest.project.clone().filter(project_has_stable_identity); + let (title, title_source) = resolve_title(events, titles); + let models = model_buckets + .into_iter() + .map(|(_, (model, totals))| model_usage(model, totals)) + .collect::>(); + let primary_model = primary_model_label(&models); + let requests = totals.requests; + + SessionRollupV1 { + schema_version: SESSION_ROLLUP_SCHEMA_VERSION.to_string(), + session_id: oldest.session.session_id.clone(), + device_id: newest.device_id.clone(), + provider: newest.provider.clone(), + source_id: newest.source_id.clone(), + provider_account_id, + started_at, + ended_at, + duration_seconds, + usage: totals.into_counts(), + requests, + cost: CostInfo { + currency: "USD".to_string(), + estimated_api_equivalent_usd: estimated_cost.cents_rounded(), + provider_reported_usd: has_provider_reported + .then(|| provider_reported_cost.cents_rounded()) + .flatten(), + estimated_api_equivalent_micro_usd: estimated_cost.micro_usd(), + provider_reported_micro_usd: has_provider_reported + .then(|| provider_reported_cost.micro_usd()) + .flatten(), + pricing_source: Some("local_rollup".to_string()), + pricing_version: None, + confidence: Confidence::Medium, + }, + models, + primary_model, + total_messages: saw_messages + .then_some(total_messages) + .filter(|value| *value > 0), + user_messages: (user_messages > 0).then_some(user_messages), + assistant_messages: (assistant_messages > 0).then_some(assistant_messages), + developer_messages: (developer_messages > 0).then_some(developer_messages), + project, + title, + title_source, + updated_at, + } +} + +fn resolve_title( + events: &[UsageEvent], + titles: &BTreeMap, +) -> (Option, Option) { + if let Some(title) = events + .iter() + .rev() + .find_map(|event| acceptable_session_title(event.session.title.as_deref())) + { + return (Some(title), Some(SessionTitleSource::Event)); + } + for event in events.iter().rev() { + let Some(hash) = event.session.local_session_id_hash.as_deref() else { + continue; + }; + if let Some((title, source)) = titles.get(hash) { + return (Some(title.clone()), Some(*source)); + } + } + (None, None) +} + +fn acceptable_session_title(value: Option<&str>) -> Option { + let trimmed = value?.trim(); + if trimmed.is_empty() || task_title_is_generic(Some(trimmed)) { + None + } else { + Some(trimmed.to_string()) + } +} + +fn remember_hashed_title( + titles: &mut BTreeMap, + wanted: &BTreeSet<&str>, + raw_id: &str, + title: Option<&str>, + source: SessionTitleSource, +) { + let Some(title) = acceptable_session_title(title) else { + return; + }; + let hash = hash_text(raw_id); + if !wanted.contains(hash.as_str()) { + return; + } + titles.insert(hash, (title, source)); +} + +fn session_rollup_content_hash(rollup: &SessionRollupV1) -> Result { + let mut comparable = rollup.clone(); + comparable.updated_at = DateTime::::UNIX_EPOCH; + Ok(hash_text(&serde_json::to_string(&comparable)?)) +} + +fn session_matches_project(rollup: &SessionRollupV1, project: Option<&str>) -> bool { + let Some(project) = project.map(str::trim).filter(|value| !value.is_empty()) else { + return true; + }; + let Some(info) = rollup.project.as_ref() else { + return false; + }; + if statsai_core::daily_rollup_project_key(Some(info)) == project || info.project_id == project { + return true; + } + [ + info.project_label.as_deref(), + info.repo_label.as_deref(), + info.path_label.as_deref(), + ] + .into_iter() + .flatten() + .any(|label| label.eq_ignore_ascii_case(project)) +} + +fn sort_sessions(sessions: &mut [SessionRollupV1], sort: SessionSort) { + sessions.sort_by(|left, right| { + let metric = match sort { + SessionSort::Started => std::cmp::Ordering::Equal, + SessionSort::Tokens => right + .usage + .computed_total() + .cmp(&left.usage.computed_total()), + SessionSort::Cost => right + .cost + .estimated_micro_usd() + .unwrap_or(0) + .cmp(&left.cost.estimated_micro_usd().unwrap_or(0)), + SessionSort::Duration => right + .duration_seconds + .unwrap_or(0) + .cmp(&left.duration_seconds.unwrap_or(0)), + SessionSort::Messages => right + .total_messages + .unwrap_or(0) + .cmp(&left.total_messages.unwrap_or(0)), + }; + metric + .then_with(|| right.started_at.cmp(&left.started_at)) + .then_with(|| left.session_id.cmp(&right.session_id)) + }); +} + +fn session_stats(rollups: &[SessionRollupV1]) -> SessionStats { + let sessions = rollups.len() as u64; + let total_tokens = rollups.iter().fold(0u64, |sum, rollup| { + sum.saturating_add(rollup.usage.computed_total()) + }); + let total_cost_micro_usd = rollups.iter().fold(0i64, |sum, rollup| { + sum.saturating_add(rollup.cost.estimated_micro_usd().unwrap_or(0)) + }); + let total_messages = rollups.iter().fold(0u64, |sum, rollup| { + sum.saturating_add(rollup.total_messages.unwrap_or(0)) + }); + let message_samples = rollups + .iter() + .filter_map(|rollup| rollup.total_messages) + .collect::>(); + let avg_messages = (!message_samples.is_empty()) + .then(|| message_samples.iter().sum::() as f64 / message_samples.len() as f64); + let mut token_samples = rollups + .iter() + .map(|rollup| rollup.usage.computed_total()) + .collect::>(); + let median_tokens = median_u64(&mut token_samples); + let mut durations = rollups + .iter() + .filter_map(|rollup| rollup.duration_seconds) + .collect::>(); + let mut model_tokens: BTreeMap<&str, u64> = BTreeMap::new(); + for rollup in rollups { + if let Some(model) = rollup.primary_model.as_deref() { + let tokens = rollup.usage.computed_total(); + let entry = model_tokens.entry(model).or_default(); + *entry = (*entry).saturating_add(tokens); + } + } + let top_model = model_tokens + .into_iter() + .max_by(|left, right| left.1.cmp(&right.1).then_with(|| right.0.cmp(left.0))) + .map(|(model, _)| model.to_string()); + SessionStats { + sessions, + total_tokens, + total_cost_micro_usd, + avg_tokens: (sessions > 0).then_some(total_tokens / sessions), + median_tokens, + avg_messages, + median_duration_seconds: median_u64(&mut durations), + total_messages, + top_model, + } +} + +fn median_u64(values: &mut [u64]) -> Option { + if values.is_empty() { + return None; + } + values.sort_unstable(); + let mid = values.len() / 2; + if values.len() % 2 == 1 { + Some(values[mid]) + } else { + Some(values[mid - 1].saturating_add(values[mid]) / 2) + } +} + +fn primary_model_label(models: &[SummaryModelUsage]) -> Option { + models + .iter() + .filter_map(|usage| { + model_label(&usage.model).map(|label| (label, usage.usage.computed_total())) + }) + .max_by(|left, right| left.1.cmp(&right.1).then_with(|| right.0.cmp(&left.0))) + .map(|(label, _)| label) +} + +fn model_label(model: &ModelInfo) -> Option { + model + .normalized_name + .clone() + .or(model.name.clone()) + .or(model.provider_model_id.clone()) + .filter(|value| !value.trim().is_empty()) +} + +fn model_usage(model: ModelInfo, totals: ModelTotals) -> SummaryModelUsage { + SummaryModelUsage { + model, + usage: UsageCounts { + input_tokens: Some(totals.input_tokens), + output_tokens: Some(totals.output_tokens), + cache_creation_tokens: Some(totals.cache_creation_tokens), + cache_creation_5m_tokens: Some(totals.cache_creation_5m_tokens), + cache_creation_1h_tokens: Some(totals.cache_creation_1h_tokens), + cache_read_tokens: Some(totals.cache_read_tokens), + reasoning_tokens: Some(totals.reasoning_tokens), + total_tokens: Some(totals.total_tokens), + requests: Some(totals.requests), + local_prompt_eval_tokens: None, + local_eval_tokens: None, + }, + cost: CostInfo { + currency: "USD".to_string(), + estimated_api_equivalent_usd: totals.estimated_cost.cents_rounded(), + provider_reported_usd: totals + .has_provider_reported + .then(|| totals.provider_reported_cost.cents_rounded()) + .flatten(), + estimated_api_equivalent_micro_usd: totals.estimated_cost.micro_usd(), + provider_reported_micro_usd: totals + .has_provider_reported + .then(|| totals.provider_reported_cost.micro_usd()) + .flatten(), + pricing_source: Some("local_rollup".to_string()), + pricing_version: None, + confidence: Confidence::Medium, + }, + metrics: None, + } +} + +fn session_model_key(model: &ModelInfo) -> String { + format!( + "{}|{}|{}|{}|{}|{}", + model.normalized_name.as_deref().unwrap_or(""), + model.provider_model_id.as_deref().unwrap_or(""), + model.name.as_deref().unwrap_or(""), + model.speed.as_deref().unwrap_or(""), + model + .reasoning_level + .as_ref() + .map(|level| level.as_str()) + .unwrap_or(""), + model.reasoning_level_raw.as_deref().unwrap_or("") + ) +} + +#[derive(Debug, Default)] +struct UsageTotals { + input_tokens: u64, + output_tokens: u64, + cache_creation_tokens: u64, + cache_creation_5m_tokens: u64, + cache_creation_1h_tokens: u64, + cache_read_tokens: u64, + reasoning_tokens: u64, + total_tokens: u64, + requests: u64, +} + +impl UsageTotals { + fn add(&mut self, usage: &UsageCounts) { + self.input_tokens = self + .input_tokens + .saturating_add(usage.input_tokens.unwrap_or(0)); + self.output_tokens = self + .output_tokens + .saturating_add(usage.output_tokens.unwrap_or(0)); + self.cache_creation_tokens = self + .cache_creation_tokens + .saturating_add(usage.cache_creation_tokens.unwrap_or(0)); + self.cache_creation_5m_tokens = self + .cache_creation_5m_tokens + .saturating_add(usage.cache_creation_5m_tokens.unwrap_or(0)); + self.cache_creation_1h_tokens = self + .cache_creation_1h_tokens + .saturating_add(usage.cache_creation_1h_tokens.unwrap_or(0)); + self.cache_read_tokens = self + .cache_read_tokens + .saturating_add(usage.cache_read_tokens.unwrap_or(0)); + self.reasoning_tokens = self + .reasoning_tokens + .saturating_add(usage.reasoning_tokens.unwrap_or(0)); + self.total_tokens = self.total_tokens.saturating_add(usage.computed_total()); + } + + fn into_counts(self) -> UsageCounts { + UsageCounts { + input_tokens: Some(self.input_tokens), + output_tokens: Some(self.output_tokens), + cache_creation_tokens: Some(self.cache_creation_tokens), + cache_creation_5m_tokens: Some(self.cache_creation_5m_tokens), + cache_creation_1h_tokens: Some(self.cache_creation_1h_tokens), + cache_read_tokens: Some(self.cache_read_tokens), + reasoning_tokens: Some(self.reasoning_tokens), + total_tokens: Some(self.total_tokens), + requests: Some(self.requests), + local_prompt_eval_tokens: None, + local_eval_tokens: None, + } + } +} + +#[derive(Debug, Default)] +struct ModelTotals { + input_tokens: u64, + output_tokens: u64, + cache_creation_tokens: u64, + cache_creation_5m_tokens: u64, + cache_creation_1h_tokens: u64, + cache_read_tokens: u64, + reasoning_tokens: u64, + total_tokens: u64, + requests: u64, + estimated_cost: CostAccumulator, + provider_reported_cost: CostAccumulator, + has_provider_reported: bool, +} + +impl ModelTotals { + fn add(&mut self, usage: &UsageCounts, requests: u64, cost: &CostInfo) { + self.input_tokens = self + .input_tokens + .saturating_add(usage.input_tokens.unwrap_or(0)); + self.output_tokens = self + .output_tokens + .saturating_add(usage.output_tokens.unwrap_or(0)); + self.cache_creation_tokens = self + .cache_creation_tokens + .saturating_add(usage.cache_creation_tokens.unwrap_or(0)); + self.cache_creation_5m_tokens = self + .cache_creation_5m_tokens + .saturating_add(usage.cache_creation_5m_tokens.unwrap_or(0)); + self.cache_creation_1h_tokens = self + .cache_creation_1h_tokens + .saturating_add(usage.cache_creation_1h_tokens.unwrap_or(0)); + self.cache_read_tokens = self + .cache_read_tokens + .saturating_add(usage.cache_read_tokens.unwrap_or(0)); + self.reasoning_tokens = self + .reasoning_tokens + .saturating_add(usage.reasoning_tokens.unwrap_or(0)); + self.total_tokens = self.total_tokens.saturating_add(usage.computed_total()); + self.requests = self.requests.saturating_add(requests); + self.estimated_cost.add_estimated(cost); + if cost.provider_reported_micro_usd.is_some() || cost.provider_reported_usd.is_some() { + self.provider_reported_cost + .add_values(cost.provider_reported_micro_usd, cost.provider_reported_usd); + self.has_provider_reported = true; + } + } +} diff --git a/crates/statsai-store/src/tasks/buckets.rs b/crates/statsai-store/src/tasks/buckets.rs index 131bfb4..be053f9 100644 --- a/crates/statsai-store/src/tasks/buckets.rs +++ b/crates/statsai-store/src/tasks/buckets.rs @@ -3,8 +3,23 @@ use super::*; impl Store { pub fn replace_task_bucket_snapshot(&self, snapshot: &TaskBucketSnapshot) -> Result<()> { self.with_immediate_transaction(|| { + let mut raw_session_ids = + self.task_span_raw_session_ids_for_bucket(&snapshot.project_bucket)?; self.delete_task_bucket_snapshot_in_tx(&snapshot.project_bucket)?; self.upsert_task_spans_in_tx(&snapshot.spans)?; + for span in &snapshot.spans { + if let Some(session_id) = span + .session_id + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + { + raw_session_ids.insert(session_id.to_string()); + } + } + self.refresh_session_rollups_for_raw_ids( + &raw_session_ids.into_iter().collect::>(), + )?; self.insert_work_items_in_tx(&snapshot.work_items, &snapshot.members)?; let mut changed_buckets = BTreeSet::new(); changed_buckets.insert(snapshot.project_bucket.clone()); diff --git a/crates/statsai-store/src/tasks/spans.rs b/crates/statsai-store/src/tasks/spans.rs index 580b306..090ee7b 100644 --- a/crates/statsai-store/src/tasks/spans.rs +++ b/crates/statsai-store/src/tasks/spans.rs @@ -142,6 +142,14 @@ impl Store { link_stmt.execute(params![&span.span_id.0, &event_id.0])?; } } + drop(span_stmt); + drop(delete_links); + drop(link_stmt); + let raw_session_ids = spans + .iter() + .filter_map(|span| span.session_id.clone()) + .collect::>(); + self.refresh_session_rollups_for_raw_ids(&raw_session_ids)?; Ok(changed) } diff --git a/crates/statsai-store/src/tests/rollups/mod.rs b/crates/statsai-store/src/tests/rollups/mod.rs index ec2a2be..5bcff87 100644 --- a/crates/statsai-store/src/tests/rollups/mod.rs +++ b/crates/statsai-store/src/tests/rollups/mod.rs @@ -2,6 +2,7 @@ pub(super) use super::support::*; pub(crate) use super::*; mod build; +mod sessions; #[test] fn renaming_a_repository_keeps_one_rollup_for_the_day() { diff --git a/crates/statsai-store/src/tests/rollups/sessions.rs b/crates/statsai-store/src/tests/rollups/sessions.rs new file mode 100644 index 0000000..ebb1ca5 --- /dev/null +++ b/crates/statsai-store/src/tests/rollups/sessions.rs @@ -0,0 +1,327 @@ +use super::*; +use statsai_core::{ + hash_text, task_title_is_generic, IdentitySource, RuntimeInfo, SessionTitleSource, TaskSpan, + TaskSpanId, TASK_SPAN_SCHEMA_VERSION, +}; + +fn stamp_session(event: &mut UsageEvent, raw_id: &str) { + let hash = hash_text(raw_id); + event.session.session_id = format!("session_{}", &hash[..24]); + event.session.local_session_id_hash = Some(hash); +} + +fn message_runtime(user: u64, assistant: u64) -> RuntimeInfo { + RuntimeInfo { + runtime_name: None, + host_id: None, + latency_ms: None, + latency_source: None, + time_to_first_token_ms: None, + prompt_eval_duration_ms: None, + eval_duration_ms: None, + total_messages: Some(user + assistant), + user_messages: Some(user), + assistant_messages: Some(assistant), + developer_messages: None, + } +} + +#[test] +fn session_rollup_sums_mixed_events() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-mixed"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 2, 9, 0, 0) + .single() + .expect("start"); + let later = start + chrono::Duration::minutes(12); + + let mut first = test_store_event(&source, start, "first"); + stamp_session(&mut first, "raw-session"); + first.session.ended_at = None; + first.usage.requests = Some(2); + first.usage.input_tokens = Some(10); + first.usage.output_tokens = Some(4); + first.usage.total_tokens = Some(14); + first.model = Some(ModelInfo { + normalized_name: Some("gpt-5".to_string()), + ..ModelInfo::default() + }); + first.runtime = Some(message_runtime(1, 1)); + first.cost.estimated_api_equivalent_micro_usd = Some(20_000); + first.created_at = start; + + let mut second = test_store_event(&source, later, "second"); + stamp_session(&mut second, "raw-session"); + second.session.started_at = start; + second.session.ended_at = Some(later); + second.usage.requests = None; + second.usage.input_tokens = Some(5); + second.usage.output_tokens = Some(7); + second.usage.total_tokens = Some(12); + second.model = Some(ModelInfo { + normalized_name: Some("gpt-5-mini".to_string()), + ..ModelInfo::default() + }); + second.usage.input_tokens = Some(30); + second.usage.output_tokens = Some(1); + second.usage.total_tokens = Some(31); + second.runtime = Some(message_runtime(2, 1)); + second.cost.estimated_api_equivalent_micro_usd = Some(5_000); + second.created_at = later; + + assert!(store.insert_event(&first).expect("insert first")); + assert!(store.insert_event(&second).expect("insert second")); + + let rollups = store + .session_rollups_in_period( + None, + later + chrono::Duration::days(1), + &SessionFilter::default(), + SessionSort::Started, + 10, + 0, + ) + .expect("rollups"); + assert_eq!(rollups.len(), 1); + let rollup = &rollups[0]; + assert_eq!(rollup.schema_version, "session_rollup.v1"); + assert_eq!(rollup.session_id, first.session.session_id); + assert_eq!(rollup.usage.input_tokens, Some(40)); + assert_eq!(rollup.usage.output_tokens, Some(5)); + assert_eq!(rollup.usage.computed_total(), 45); + assert_eq!(rollup.requests, 3); + assert_eq!(rollup.cost.estimated_micro_usd(), Some(25_000)); + assert_eq!(rollup.user_messages, Some(3)); + assert_eq!(rollup.assistant_messages, Some(2)); + assert_eq!(rollup.total_messages, Some(5)); + assert_eq!(rollup.primary_model.as_deref(), Some("gpt-5-mini")); + assert_eq!(rollup.started_at, start); + assert_eq!(rollup.ended_at, later); + assert_eq!(rollup.duration_seconds, Some(12 * 60)); + assert_eq!(rollup.models.len(), 2); +} + +#[test] +fn session_rollup_refreshes_on_insert_and_delete() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-refresh"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 3, 9, 0, 0) + .single() + .expect("start"); + let mut kept = test_store_event(&source, start, "kept"); + stamp_session(&mut kept, "refresh-session"); + kept.usage.total_tokens = Some(10); + kept.parse_evidence = Some(ParseEvidence { + event_key_version: "test".to_string(), + source_file_path_hash: Some("file-kept".to_string()), + source_line_number: None, + source_record_id: None, + model_inferred: false, + timestamp_inferred: false, + account_identity_source: IdentitySource::Unresolved, + }); + let mut dropped = test_store_event(&source, start + chrono::Duration::minutes(1), "dropped"); + stamp_session(&mut dropped, "refresh-session"); + dropped.session.started_at = start; + dropped.usage.total_tokens = Some(7); + dropped.parse_evidence = Some(ParseEvidence { + event_key_version: "test".to_string(), + source_file_path_hash: Some("file-dropped".to_string()), + source_line_number: None, + source_record_id: None, + model_inferred: false, + timestamp_inferred: false, + account_identity_source: IdentitySource::Unresolved, + }); + store.insert_event(&kept).expect("kept"); + store.insert_event(&dropped).expect("dropped"); + let before = store.dirty_session_rollups().expect("dirty"); + assert_eq!(before.len(), 1); + assert_eq!(before[0].usage.computed_total(), 17); + + let impact = store + .delete_events_for_source_file_hashes(&source.source_id, &["file-dropped".to_string()]) + .expect("delete"); + assert_eq!(impact.deleted, 1); + let after = store.dirty_session_rollups().expect("after"); + assert_eq!(after.len(), 1); + assert_eq!(after[0].usage.computed_total(), 10); + + store + .delete_events_for_sources(std::slice::from_ref(&source.source_id)) + .expect("delete source"); + assert!(store.dirty_session_rollups().expect("gone").is_empty()); +} + +#[test] +fn session_titles_prefer_event_then_task_span_then_archive() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-titles"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 4, 9, 0, 0) + .single() + .expect("start"); + let raw_id = "provider-session-9"; + let mut event = test_store_event(&source, start, "titled"); + stamp_session(&mut event, raw_id); + store.insert_event(&event).expect("insert"); + + let generic = "hello"; + assert!(task_title_is_generic(Some(generic))); + store + .upsert_task_spans(&[task_span(&source, raw_id, generic, start)]) + .expect("generic span"); + let unlabeled = store.dirty_session_rollups().expect("unlabeled"); + assert_eq!(unlabeled[0].title, None); + + let title = "Repair the session indexer"; + assert!(!task_title_is_generic(Some(title))); + store + .upsert_task_spans(&[task_span(&source, raw_id, title, start)]) + .expect("span"); + let from_span = store.dirty_session_rollups().expect("span title"); + assert_eq!(from_span[0].title.as_deref(), Some(title)); + assert_eq!( + from_span[0].title_source, + Some(SessionTitleSource::TaskSpan) + ); + + event.session.title = Some("Event title wins".to_string()); + store.insert_event(&event).expect("retitle"); + let from_event = store.dirty_session_rollups().expect("event title"); + assert_eq!(from_event[0].title.as_deref(), Some("Event title wins")); + assert_eq!(from_event[0].title_source, Some(SessionTitleSource::Event)); +} + +#[test] +fn session_queries_filter_period_and_sort() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "claude_code", + "test", + "0", + Path::new("/tmp/session-query"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let early = Utc + .with_ymd_and_hms(2026, 6, 1, 8, 0, 0) + .single() + .expect("early"); + let late = Utc + .with_ymd_and_hms(2026, 6, 10, 8, 0, 0) + .single() + .expect("late"); + let mut small = test_store_event(&source, early, "small"); + stamp_session(&mut small, "small-session"); + small.provider = "claude_code".to_string(); + small.usage.total_tokens = Some(5); + small.session.ended_at = Some(early + chrono::Duration::seconds(30)); + let mut large = test_store_event(&source, late, "large"); + stamp_session(&mut large, "large-session"); + large.provider = "claude_code".to_string(); + large.usage.total_tokens = Some(50); + large.session.ended_at = Some(late + chrono::Duration::seconds(90)); + store.insert_event(&small).expect("small"); + store.insert_event(&large).expect("large"); + + let filter = SessionFilter { + provider: Some("claude_code".to_string()), + ..SessionFilter::default() + }; + let page = store + .session_rollups_in_period( + Some(Utc.with_ymd_and_hms(2026, 6, 5, 0, 0, 0).unwrap()), + late + chrono::Duration::days(1), + &filter, + SessionSort::Tokens, + 10, + 0, + ) + .expect("page"); + assert_eq!(page.len(), 1); + assert_eq!(page[0].usage.computed_total(), 50); + + let sorted = store + .session_rollups_in_period(None, late, &filter, SessionSort::Tokens, 10, 0) + .expect("sorted"); + assert_eq!(sorted.len(), 2); + assert_eq!(sorted[0].usage.computed_total(), 50); + assert_eq!(sorted[1].usage.computed_total(), 5); + let stats = store + .session_stats_in_period(None, late, &filter) + .expect("stats"); + assert_eq!(stats.sessions, 2); + assert_eq!(stats.total_tokens, 55); + assert_eq!(stats.avg_tokens, Some(27)); + assert_eq!(stats.median_tokens, Some(27)); + assert_eq!(stats.median_duration_seconds, Some(60)); +} + +fn task_span( + source: &statsai_core::SourceLocation, + raw_session_id: &str, + title: &str, + started_at: chrono::DateTime, +) -> TaskSpan { + TaskSpan { + schema_version: TASK_SPAN_SCHEMA_VERSION.to_string(), + span_id: TaskSpanId(format!("span-{raw_session_id}-{title}")), + provider: source.provider.clone(), + source_id: source.source_id.clone(), + span_kind: "test".to_string(), + source_record_id: None, + source_file_path_hash: None, + summary_id: None, + session_id: Some(raw_session_id.to_string()), + thread_id: None, + title: title.to_string(), + normalized_title: statsai_core::normalize_task_title(title), + title_source: Some("test".to_string()), + summary_preview: None, + todo_excerpt: None, + issue_keys: Vec::new(), + branch_family: None, + project_bucket: "bucket".to_string(), + project: None, + git: None, + usage: UsageCounts::default(), + estimated_cost_usd: None, + estimated_cost_micro_usd: None, + event_count: 0, + has_usage_evidence: false, + total_messages: 0, + user_messages: 0, + assistant_messages: 0, + developer_messages: 0, + linked_event_ids: Vec::new(), + confidence: Confidence::Medium, + is_meta: task_title_is_generic(Some(title)), + started_at, + ended_at: Some(started_at), + duration_seconds: Some(0), + } +} diff --git a/crates/statsai/src/cli/args/mod.rs b/crates/statsai/src/cli/args/mod.rs index 027d8e6..0d80179 100644 --- a/crates/statsai/src/cli/args/mod.rs +++ b/crates/statsai/src/cli/args/mod.rs @@ -1,4 +1,4 @@ -use clap::{Args, Parser, Subcommand}; +use clap::{Args, Parser, Subcommand, ValueEnum}; use statsai::snapshot; use std::path::PathBuf; use std::str::FromStr; @@ -8,6 +8,7 @@ mod activity; mod quota; mod report; mod scan; +mod sessions; mod source; mod subscription; mod sync; @@ -18,6 +19,7 @@ pub(crate) use activity::*; pub(crate) use quota::*; pub(crate) use report::*; pub(crate) use scan::*; +pub(crate) use sessions::*; pub(crate) use source::*; pub(crate) use subscription::*; pub(crate) use sync::*; @@ -46,6 +48,8 @@ pub(crate) enum Command { Scan(ScanCommand), #[command(about = "Show usage reports (weekly, monthly, all-time, or a date range)")] Report(ReportCommand), + #[command(about = "List local session rollups")] + Sessions(SessionsCommand), #[command(about = "Manage configured source paths")] Source(SourceCommand), #[command(about = "List canonical provider accounts")] diff --git a/crates/statsai/src/cli/args/sessions.rs b/crates/statsai/src/cli/args/sessions.rs new file mode 100644 index 0000000..03a5fb9 --- /dev/null +++ b/crates/statsai/src/cli/args/sessions.rs @@ -0,0 +1,43 @@ +use super::*; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +pub(crate) enum SessionSortArg { + Started, + Tokens, + Cost, + Duration, + Messages, +} + +impl SessionSortArg { + pub(crate) fn store_sort(self) -> statsai_store::SessionSort { + match self { + Self::Started => statsai_store::SessionSort::Started, + Self::Tokens => statsai_store::SessionSort::Tokens, + Self::Cost => statsai_store::SessionSort::Cost, + Self::Duration => statsai_store::SessionSort::Duration, + Self::Messages => statsai_store::SessionSort::Messages, + } + } +} + +#[derive(Debug, Args)] +pub(crate) struct SessionsCommand { + #[arg( + long, + help = "Range start (YYYY-MM-DD or RFC3339). Defaults to the last 7 days" + )] + pub(crate) from: Option, + #[arg(long, help = "Range end (YYYY-MM-DD or RFC3339). Defaults to now")] + pub(crate) to: Option, + #[arg(long, help = "Provider filter")] + pub(crate) provider: Option, + #[arg(long, help = "Project id, key, or label")] + pub(crate) project: Option, + #[arg(long, value_enum, default_value = "started", help = "Sort descending")] + pub(crate) sort: SessionSortArg, + #[arg(long, default_value_t = 50, help = "Maximum sessions to print")] + pub(crate) limit: usize, + #[arg(long, help = "Output as JSON")] + pub(crate) json: bool, +} diff --git a/crates/statsai/src/cli/dispatch.rs b/crates/statsai/src/cli/dispatch.rs index 90ae129..a8ddb6a 100644 --- a/crates/statsai/src/cli/dispatch.rs +++ b/crates/statsai/src/cli/dispatch.rs @@ -26,6 +26,7 @@ pub(crate) fn run() -> Result<()> { match command { Command::Scan(command) => scan(command, &store, &device_id), Command::Report(command) => report(command, &store), + Command::Sessions(command) => sessions(command, &store), Command::Source(command) => source(command, &store, &device_id), Command::Account(command) => account(command, &store), Command::Subscription(command) => subscription(command, &store), @@ -59,6 +60,7 @@ pub(crate) fn command_reprices_persisted_usage(command: &Command) -> bool { command, Command::Scan(_) | Command::Report(_) + | Command::Sessions(_) | Command::Import(_) | Command::Export(_) | Command::Task(_) diff --git a/crates/statsai/src/cli/mod.rs b/crates/statsai/src/cli/mod.rs index ab38a50..e3e365a 100644 --- a/crates/statsai/src/cli/mod.rs +++ b/crates/statsai/src/cli/mod.rs @@ -12,6 +12,7 @@ pub(super) mod report; pub(super) mod scan; pub(super) mod schema; pub(super) mod service; +pub(super) mod sessions; pub(super) mod source; pub(super) mod status; pub(super) mod store_admin; @@ -34,6 +35,7 @@ pub(crate) use report::*; pub(crate) use scan::*; pub(crate) use schema::*; pub(crate) use service::*; +pub(crate) use sessions::*; pub(crate) use source::*; pub(crate) use status::*; pub(crate) use store_admin::*; diff --git a/crates/statsai/src/cli/sessions.rs b/crates/statsai/src/cli/sessions.rs new file mode 100644 index 0000000..89dc073 --- /dev/null +++ b/crates/statsai/src/cli/sessions.rs @@ -0,0 +1,151 @@ +use anyhow::Result; +use chrono::{DateTime, Utc}; +use serde_json::json; +use statsai_core::{ + micro_usd_to_cents_rounded, report_period_from_range, ReportPeriod, SessionRollupV1, +}; +use statsai_store::{SessionFilter, SessionStats, Store}; + +use super::args::SessionsCommand; +use super::format::{format_cost, format_u64, truncate_label}; +use super::source::canonical_provider; + +pub(crate) struct SessionsView { + pub(crate) label: String, + pub(crate) stats: SessionStats, + pub(crate) sessions: Vec, +} + +pub(crate) fn sessions(command: SessionsCommand, store: &Store) -> Result<()> { + let view = sessions_view(&command, store, Utc::now())?; + if command.json { + println!("{}", serde_json::to_string_pretty(&sessions_json(&view))?); + } else { + print_sessions_table(&view); + } + Ok(()) +} + +pub(crate) fn sessions_view( + command: &SessionsCommand, + store: &Store, + now: DateTime, +) -> Result { + let period = match (&command.from, &command.to) { + (None, None) => ReportPeriod::LastDays(7), + (from, to) => report_period_from_range(from.as_deref(), to.as_deref(), now)?, + }; + let (since, until) = period.published_window(now); + let provider = command + .provider + .as_deref() + .map(canonical_provider) + .transpose()?; + let filter = SessionFilter { + provider, + project_key: command.project.clone(), + account: None, + }; + let stats = store.session_stats_in_period(since, until, &filter)?; + let sessions = store.session_rollups_in_period( + since, + until, + &filter, + command.sort.store_sort(), + command.limit, + 0, + )?; + Ok(SessionsView { + label: period.label(now), + stats, + sessions, + }) +} + +fn sessions_json(view: &SessionsView) -> serde_json::Value { + json!({ + "label": view.label, + "stats": view.stats, + "sessions": view.sessions, + }) +} + +fn print_sessions_table(view: &SessionsView) { + let stats = &view.stats; + println!("statsai sessions: {}", view.label); + println!("sessions: {}", format_u64(stats.sessions)); + println!("tokens: {}", format_u64(stats.total_tokens)); + println!( + "avg tokens/session: {}", + stats + .avg_tokens + .map(format_u64) + .unwrap_or_else(|| "unknown".to_string()) + ); + println!( + "median duration: {}", + format_duration(stats.median_duration_seconds) + ); + println!("messages: {}", format_u64(stats.total_messages)); + println!( + "est. cost: {}", + format_cost(Some(micro_usd_to_cents_rounded(stats.total_cost_micro_usd))) + ); + println!( + "top model: {}", + stats.top_model.as_deref().unwrap_or("unknown") + ); + println!(); + println!( + "{:<16} {:<12} {:<18} {:<24} {:>6} {:>10} {:>10} {:>10} {:>10} {:>10}", + "started", + "provider", + "model", + "title", + "msgs", + "input", + "output", + "total", + "est_cost", + "duration" + ); + for session in &view.sessions { + println!( + "{:<16} {:<12} {:<18} {:<24} {:>6} {:>10} {:>10} {:>10} {:>10} {:>10}", + session.started_at.format("%Y-%m-%d %H:%M"), + truncate_label(&session.provider, 12), + truncate_label(session.primary_model.as_deref().unwrap_or("unknown"), 18), + truncate_label(session.title.as_deref().unwrap_or("—"), 24), + session + .total_messages + .map(format_u64) + .unwrap_or_else(|| "—".to_string()), + format_u64(session.usage.input_tokens.unwrap_or(0)), + format_u64(session.usage.output_tokens.unwrap_or(0)), + format_u64(session.usage.computed_total()), + format_cost( + session + .cost + .estimated_micro_usd() + .map(micro_usd_to_cents_rounded) + ), + format_duration(session.duration_seconds) + ); + } +} + +fn format_duration(seconds: Option) -> String { + let Some(seconds) = seconds else { + return "unknown".to_string(); + }; + let hours = seconds / 3600; + let minutes = (seconds % 3600) / 60; + let secs = seconds % 60; + if hours > 0 { + format!("{hours}h {minutes:02}m") + } else if minutes > 0 { + format!("{minutes}m {secs:02}s") + } else { + format!("{secs}s") + } +} diff --git a/crates/statsai/src/cli/tests/mod.rs b/crates/statsai/src/cli/tests/mod.rs index 159246c..bb1a76f 100644 --- a/crates/statsai/src/cli/tests/mod.rs +++ b/crates/statsai/src/cli/tests/mod.rs @@ -8,6 +8,7 @@ mod import; mod quota; mod report; mod scan; +mod sessions; mod source; mod status; mod subscription; diff --git a/crates/statsai/src/cli/tests/sessions.rs b/crates/statsai/src/cli/tests/sessions.rs new file mode 100644 index 0000000..f8d9257 --- /dev/null +++ b/crates/statsai/src/cli/tests/sessions.rs @@ -0,0 +1,100 @@ +use super::support::*; +use super::*; + +#[test] +fn sessions_cli_parses_filters() { + let cli = Cli::try_parse_from([ + "statsai", + "sessions", + "--from", + "2026-06-01", + "--to", + "2026-06-15", + "--provider", + "codex", + "--project", + "statsai", + "--sort", + "tokens", + "--limit", + "10", + "--json", + ]) + .expect("parse"); + match cli.command { + Command::Sessions(command) => { + assert_eq!(command.from.as_deref(), Some("2026-06-01")); + assert_eq!(command.to.as_deref(), Some("2026-06-15")); + assert_eq!(command.provider.as_deref(), Some("codex")); + assert_eq!(command.project.as_deref(), Some("statsai")); + assert_eq!(command.sort, SessionSortArg::Tokens); + assert_eq!(command.limit, 10); + assert!(command.json); + } + other => panic!("unexpected command {other:?}"), + } +} + +#[test] +fn sessions_view_reads_rollups_for_the_default_week() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/sessions-cli"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let now = Utc + .with_ymd_and_hms(2026, 6, 20, 12, 0, 0) + .single() + .expect("now"); + let mut recent = test_event( + "codex", + &source, + now - Duration::days(1), + None, + TokenParts::total(40), + ); + recent.session.session_id = "session_recent".to_string(); + recent.session.title = Some("Recent indexing work".to_string()); + recent.model = Some(ModelInfo { + normalized_name: Some("gpt-5".to_string()), + ..ModelInfo::default() + }); + let mut old = test_event( + "codex", + &source, + now - Duration::days(30), + None, + TokenParts::total(9), + ); + old.session.session_id = "session_old".to_string(); + store.insert_event(&recent).expect("recent"); + store.insert_event(&old).expect("old"); + + let command = SessionsCommand { + from: None, + to: None, + provider: Some("codex".to_string()), + project: None, + sort: SessionSortArg::Started, + limit: 50, + json: true, + }; + let view = sessions_view(&command, &store, now).expect("view"); + assert_eq!(view.label, "last 7 days"); + assert_eq!(view.stats.sessions, 1); + assert_eq!(view.stats.total_tokens, 40); + assert_eq!(view.stats.top_model.as_deref(), Some("gpt-5")); + assert_eq!(view.sessions.len(), 1); + assert_eq!( + view.sessions[0].title.as_deref(), + Some("Recent indexing work") + ); + assert_eq!( + view.sessions[0].title_source, + Some(statsai_core::SessionTitleSource::Event) + ); +} diff --git a/crates/statsai/tests/cli_surface/help.txt b/crates/statsai/tests/cli_surface/help.txt index ef70d41..09e46a2 100644 --- a/crates/statsai/tests/cli_surface/help.txt +++ b/crates/statsai/tests/cli_surface/help.txt @@ -6,6 +6,7 @@ Usage: statsai [OPTIONS] Commands: scan Scan local provider sources for usage events report Show usage reports (weekly, monthly, all-time, or a date range) + sessions List local session rollups source Manage configured source paths account List canonical provider accounts subscription Manage subscription periods @@ -114,6 +115,22 @@ Options: --verbose Show source paths and reasoning tokens --subscriptions Include subscription-value rows -h, --help Print help +===== statsai sessions --help ===== +List local session rollups + +Usage: statsai sessions [OPTIONS] + +Options: + --from Range start (YYYY-MM-DD or RFC3339). Defaults to the last 7 days + --store Path to SQLite store + --device-id Device identifier for multi-device sync + --to Range end (YYYY-MM-DD or RFC3339). Defaults to now + --provider Provider filter + --project Project id, key, or label + --sort Sort descending [default: started] [possible values: started, tokens, cost, duration, messages] + --limit Maximum sessions to print [default: 50] + --json Output as JSON + -h, --help Print help ===== statsai source --help ===== Manage configured source paths From fdbb922948dd9afc5d99c104e5f0adbdcdae85c0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 25 Sep 2026 19:36:51 +0000 Subject: [PATCH 02/16] Sync session rollups on sync_batch.v6 Send closed session_rollup.v1 rows when session sync is enabled, and retire hosted sessions only when the snapshot names session_rollup_ids. Co-authored-by: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> --- README.md | 7 +- crates/statsai-core/src/tests/types.rs | 103 ++++++ crates/statsai-core/src/types/session.rs | 230 ++++++++++++- crates/statsai-core/src/types/sync.rs | 33 +- crates/statsai-daemon/src/ingest.rs | 8 + crates/statsai-daemon/src/tests.rs | 56 +++ crates/statsai-store/src/accounts.rs | 15 + .../src/code_changes/tests/refresh.rs | 1 + crates/statsai-store/src/lib.rs | 5 +- crates/statsai-store/src/rollups/sessions.rs | 14 + crates/statsai-store/src/sync_state/mod.rs | 14 + .../statsai-store/src/sync_state/pending.rs | 29 +- crates/statsai-store/src/sync_state/record.rs | 20 ++ .../statsai-store/src/tests/sync_state/mod.rs | 21 ++ crates/statsai-sync/src/lib.rs | 7 + crates/statsai/src/cli/args/sync.rs | 14 +- crates/statsai/src/cli/sync/batch.rs | 51 ++- .../statsai/src/cli/sync/chunking/budget.rs | 13 + crates/statsai/src/cli/sync/chunking/mod.rs | 48 ++- .../statsai/src/cli/sync/chunking/payloads.rs | 85 +++++ crates/statsai/src/cli/sync/mod.rs | 35 +- crates/statsai/src/cli/tests/support.rs | 2 + crates/statsai/src/cli/tests/sync/batch.rs | 4 + .../src/cli/tests/sync/chunking/budget.rs | 4 + .../src/cli/tests/sync/chunking/mod.rs | 71 ++++ .../src/cli/tests/sync/chunking/payloads.rs | 2 + crates/statsai/src/cli/tests/sync/http.rs | 2 + crates/statsai/src/cli/tests/sync/mod.rs | 1 + crates/statsai/tests/cli_surface/help.txt | 2 + .../tests/cli_surface/schema-sync-batch.json | 319 ++++++++++++++++++ docs/sync-contract.md | 21 +- 31 files changed, 1214 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 8982bb7..bdc5a5e 100644 --- a/README.md +++ b/README.md @@ -156,13 +156,18 @@ dashboard. | Parse evidence and local source paths | Project labels and repository anchors | | Source text, diffs, file paths, and commit messages | Privacy-safe numeric code-change metrics | | — | Opt-in private task snapshots and verifications, including bounded task titles, summary previews, and todo excerpts | +| Session prompts, raw session ids, and message text | Opt-in session rollups: totals, duration, message counts, and a bounded title | Raw usage events and complete archived conversation records stay local and are never included in hosted sync. StatsAI does not upload full prompts, full responses, or raw provider logs. When hosted task sync is explicitly enabled with `statsai sync --include-tasks`, the current `sync_batch.v5` payload may include bounded conversation-derived task titles, summary previews, and todo -excerpts. You can inspect the exact sync contract in +excerpts. Hosted session sync is separate and off by default. +`statsai sync --include-sessions` adds `sessions` rows to `sync_batch.v6`: +hashed session ids, token and request totals, duration, message counts, and a +bounded title. `--exclude-sessions` stops sending them and leaves sessions +already hosted in place. You can inspect the exact sync contract in [`docs/sync-contract.md`](docs/sync-contract.md) and verify the resolved sync target with: diff --git a/crates/statsai-core/src/tests/types.rs b/crates/statsai-core/src/tests/types.rs index 5386219..1bc3731 100644 --- a/crates/statsai-core/src/tests/types.rs +++ b/crates/statsai-core/src/tests/types.rs @@ -133,10 +133,105 @@ fn session_rollup_uses_snake_case_contract() { assert_eq!(json["schema_version"], "session_rollup.v1"); assert_eq!(json["title_source"], "task_span"); assert_eq!(json["requests"], 1); + assert_eq!(json["duration_seconds"], 0); + assert!(json["usage"].get("cache_creation_5m_tokens").is_none()); + assert!(json["usage"].get("cache_creation_1h_tokens").is_none()); + assert!(json["cost"].get("currency").is_none()); + assert!(json["cost"].get("pricing_source").is_none()); + assert!(json["cost"].get("confidence").is_none()); let restored: SessionRollupV1 = serde_json::from_value(json).expect("deserialize"); assert_eq!(restored, rollup); } +#[test] +fn session_rollup_unknown_duration_serializes_as_zero() { + let started = DateTime::parse_from_rfc3339("2026-05-01T00:00:00Z") + .expect("start") + .with_timezone(&Utc); + let mut rollup = SessionRollupV1 { + schema_version: SESSION_ROLLUP_SCHEMA_VERSION.to_string(), + session_id: "session_abc".to_string(), + device_id: "device".to_string(), + provider: "codex".to_string(), + source_id: SourceId("src".to_string()), + provider_account_id: None, + started_at: started, + ended_at: started, + duration_seconds: None, + usage: UsageCounts::default(), + requests: 0, + cost: CostInfo { + currency: "USD".to_string(), + estimated_api_equivalent_usd: None, + provider_reported_usd: None, + estimated_api_equivalent_micro_usd: None, + provider_reported_micro_usd: None, + pricing_source: None, + pricing_version: None, + confidence: Confidence::Medium, + }, + models: Vec::new(), + primary_model: None, + total_messages: None, + user_messages: None, + assistant_messages: None, + developer_messages: None, + project: None, + title: None, + title_source: None, + updated_at: started, + }; + let json = serde_json::to_value(&rollup).expect("serialize"); + assert_eq!(json["duration_seconds"], 0); + rollup.duration_seconds = Some(0); + let restored: SessionRollupV1 = serde_json::from_value(json).expect("deserialize"); + assert_eq!(restored, rollup); +} + +#[test] +fn sync_batch_omits_empty_sessions_and_absent_session_retirement() { + let batch = SyncBatch { + schema_version: SYNC_BATCH_V6_SCHEMA_VERSION.to_string(), + batch_id: "batch-sessions".to_string(), + device_id: "device-1".to_string(), + sources: Vec::new(), + accounts: Vec::new(), + source_account_assignments: Vec::new(), + subscriptions: Vec::new(), + events: Vec::new(), + summaries: Vec::new(), + task_buckets: Vec::new(), + task_verifications: Vec::new(), + code_change_metrics: Vec::new(), + quota_cycle_contributions: Vec::new(), + account_plan_observations: Vec::new(), + account_evidence_summaries: Vec::new(), + activity_rollups: Vec::new(), + activity_coverage: Vec::new(), + sessions: Vec::new(), + authoritative_snapshot: Some(SyncAuthoritativeSnapshot::default()), + created_at: DateTime::parse_from_rfc3339("2026-05-31T10:00:00Z") + .expect("timestamp") + .with_timezone(&Utc), + }; + let json = serde_json::to_value(&batch).expect("serialize"); + assert!(json.get("sessions").is_none()); + assert!(json["authoritative_snapshot"] + .get("session_rollup_ids") + .is_none()); + + let mut retiring = batch; + retiring.authoritative_snapshot = Some(SyncAuthoritativeSnapshot { + session_rollup_ids: Some(Vec::new()), + ..SyncAuthoritativeSnapshot::default() + }); + let json = serde_json::to_value(&retiring).expect("serialize retirement"); + assert_eq!( + json["authoritative_snapshot"]["session_rollup_ids"], + serde_json::json!([]) + ); +} + #[test] fn sync_batch_without_authoritative_snapshot_remains_backward_compatible() { let batch: SyncBatch = serde_json::from_value(serde_json::json!({ @@ -172,6 +267,7 @@ fn sync_ack_v1_omits_zero_task_counters() { account_evidence_summaries: 0, activity_rollups: 0, activity_coverage: 0, + sessions: 0, }, duplicates: SyncEntityCounts { sources: 0, @@ -188,6 +284,7 @@ fn sync_ack_v1_omits_zero_task_counters() { account_evidence_summaries: 0, activity_rollups: 0, activity_coverage: 0, + sessions: 0, }, rejected: Vec::new(), }; @@ -200,6 +297,8 @@ fn sync_ack_v1_omits_zero_task_counters() { assert!(json["duplicates"].get("task_buckets").is_none()); assert!(json["duplicates"].get("task_verifications").is_none()); assert!(json["duplicates"].get("code_change_metrics").is_none()); + assert!(json["accepted"].get("sessions").is_none()); + assert!(json["duplicates"].get("sessions").is_none()); } #[test] @@ -222,6 +321,7 @@ fn sync_ack_v3_keeps_nonzero_task_and_code_change_counters() { account_evidence_summaries: 0, activity_rollups: 0, activity_coverage: 0, + sessions: 0, }, duplicates: SyncEntityCounts { sources: 0, @@ -238,6 +338,7 @@ fn sync_ack_v3_keeps_nonzero_task_and_code_change_counters() { account_evidence_summaries: 0, activity_rollups: 0, activity_coverage: 0, + sessions: 0, }, rejected: Vec::new(), }; @@ -272,6 +373,7 @@ fn sync_ack_v4_keeps_nonzero_quota_cycle_counters() { account_evidence_summaries: 0, activity_rollups: 0, activity_coverage: 0, + sessions: 0, }, duplicates: SyncEntityCounts { sources: 0, @@ -288,6 +390,7 @@ fn sync_ack_v4_keeps_nonzero_quota_cycle_counters() { account_evidence_summaries: 0, activity_rollups: 0, activity_coverage: 0, + sessions: 0, }, rejected: Vec::new(), }; diff --git a/crates/statsai-core/src/types/session.rs b/crates/statsai-core/src/types/session.rs index 1b8b7d2..c34c157 100644 --- a/crates/statsai-core/src/types/session.rs +++ b/crates/statsai-core/src/types/session.rs @@ -1,5 +1,5 @@ -use super::event::{CostInfo, ProjectInfo, UsageCounts}; -use super::summary::SummaryModelUsage; +use super::event::{Confidence, CostInfo, ModelInfo, ProjectInfo, UsageCounts}; +use super::summary::{SummaryModelMetrics, SummaryModelUsage}; use crate::ids::{ProviderAccountId, SourceId}; use chrono::{DateTime, Utc}; use schemars::JsonSchema; @@ -31,10 +31,26 @@ pub struct SessionRollupV1 { pub provider_account_id: Option, pub started_at: DateTime, pub ended_at: DateTime, + /// Always a non-negative integer on the wire. Unknown durations serialize as 0 + /// because `session_rollup.v1` rejects null. + #[serde( + serialize_with = "serialize_duration_seconds", + deserialize_with = "deserialize_duration_seconds" + )] + #[schemars(with = "u64")] pub duration_seconds: Option, + /// Closed usage object. Cache-lifetime splits stay local; the ingest allowlist + /// does not include them. + #[serde(with = "session_usage_serde")] + #[schemars(with = "SessionUsageWire")] pub usage: UsageCounts, pub requests: u64, + /// Closed cost object. Currency, pricing source, and confidence stay local. + #[serde(with = "session_cost_serde")] + #[schemars(with = "SessionCostWire")] pub cost: CostInfo, + #[serde(with = "session_models_serde")] + #[schemars(with = "Vec")] pub models: Vec, pub primary_model: Option, pub total_messages: Option, @@ -46,3 +62,213 @@ pub struct SessionRollupV1 { pub title_source: Option, pub updated_at: DateTime, } + +/// Usage keys accepted on `session_rollup.v1`. Five-minute and one-hour cache +/// splits are folded into `cache_creation_tokens` before this leaves the device. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema, Default)] +pub struct SessionUsageWire { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub local_prompt_eval_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub local_eval_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub requests: Option, +} + +/// Cost keys accepted on `session_rollup.v1`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema, Default)] +pub struct SessionCostWire { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_reported_usd: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub estimated_api_equivalent_usd: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_reported_micro_usd: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub estimated_api_equivalent_micro_usd: Option, +} + +/// One model row on `session_rollup.v1`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct SessionModelWire { + pub model: ModelInfo, + pub usage: SessionUsageWire, + pub cost: SessionCostWire, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metrics: Option, +} + +impl From<&UsageCounts> for SessionUsageWire { + fn from(usage: &UsageCounts) -> Self { + Self { + total_tokens: usage.total_tokens, + input_tokens: usage.input_tokens, + output_tokens: usage.output_tokens, + cache_creation_tokens: usage.cache_creation_tokens, + cache_read_tokens: usage.cache_read_tokens, + reasoning_tokens: usage.reasoning_tokens, + local_prompt_eval_tokens: usage.local_prompt_eval_tokens, + local_eval_tokens: usage.local_eval_tokens, + requests: usage.requests, + } + } +} + +impl From for UsageCounts { + fn from(usage: SessionUsageWire) -> Self { + Self { + input_tokens: usage.input_tokens, + output_tokens: usage.output_tokens, + cache_creation_tokens: usage.cache_creation_tokens, + cache_creation_5m_tokens: None, + cache_creation_1h_tokens: None, + cache_read_tokens: usage.cache_read_tokens, + reasoning_tokens: usage.reasoning_tokens, + total_tokens: usage.total_tokens, + requests: usage.requests, + local_prompt_eval_tokens: usage.local_prompt_eval_tokens, + local_eval_tokens: usage.local_eval_tokens, + } + } +} + +impl From<&CostInfo> for SessionCostWire { + fn from(cost: &CostInfo) -> Self { + Self { + provider_reported_usd: cost.provider_reported_usd, + estimated_api_equivalent_usd: cost.estimated_api_equivalent_usd, + provider_reported_micro_usd: cost.provider_reported_micro_usd, + estimated_api_equivalent_micro_usd: cost.estimated_api_equivalent_micro_usd, + } + } +} + +impl From for CostInfo { + fn from(cost: SessionCostWire) -> Self { + Self { + currency: "USD".to_string(), + estimated_api_equivalent_usd: cost.estimated_api_equivalent_usd, + provider_reported_usd: cost.provider_reported_usd, + estimated_api_equivalent_micro_usd: cost.estimated_api_equivalent_micro_usd, + provider_reported_micro_usd: cost.provider_reported_micro_usd, + pricing_source: None, + pricing_version: None, + confidence: Confidence::Medium, + } + } +} + +impl From<&SummaryModelUsage> for SessionModelWire { + fn from(usage: &SummaryModelUsage) -> Self { + Self { + model: usage.model.clone(), + usage: SessionUsageWire::from(&usage.usage), + cost: SessionCostWire::from(&usage.cost), + metrics: usage.metrics.clone(), + } + } +} + +impl From for SummaryModelUsage { + fn from(usage: SessionModelWire) -> Self { + Self { + model: usage.model, + usage: UsageCounts::from(usage.usage), + cost: CostInfo::from(usage.cost), + metrics: usage.metrics, + } + } +} + +fn serialize_duration_seconds(value: &Option, serializer: S) -> Result +where + S: serde::Serializer, +{ + serializer.serialize_u64(value.unwrap_or(0)) +} + +fn deserialize_duration_seconds<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + Option::::deserialize(deserializer) +} + +mod session_usage_serde { + use super::{SessionUsageWire, UsageCounts}; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + pub(super) fn serialize(usage: &UsageCounts, serializer: S) -> Result + where + S: Serializer, + { + SessionUsageWire::from(usage).serialize(serializer) + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + Ok(UsageCounts::from(SessionUsageWire::deserialize( + deserializer, + )?)) + } +} + +mod session_cost_serde { + use super::{CostInfo, SessionCostWire}; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + pub(super) fn serialize(cost: &CostInfo, serializer: S) -> Result + where + S: Serializer, + { + SessionCostWire::from(cost).serialize(serializer) + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + Ok(CostInfo::from(SessionCostWire::deserialize(deserializer)?)) + } +} + +mod session_models_serde { + use super::{SessionModelWire, SummaryModelUsage}; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + pub(super) fn serialize( + models: &[SummaryModelUsage], + serializer: S, + ) -> Result + where + S: Serializer, + { + let wire = models + .iter() + .map(SessionModelWire::from) + .collect::>(); + wire.serialize(serializer) + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let wire = Vec::::deserialize(deserializer)?; + Ok(wire.into_iter().map(SummaryModelUsage::from).collect()) + } +} diff --git a/crates/statsai-core/src/types/sync.rs b/crates/statsai-core/src/types/sync.rs index 65edf24..c4d7997 100644 --- a/crates/statsai-core/src/types/sync.rs +++ b/crates/statsai-core/src/types/sync.rs @@ -1,10 +1,10 @@ use crate::{ project_contains_file_paths, project_has_stable_identity, AccountEvidenceSummaryV1, AccountPlanProjectionV1, ActivityCoverageV1, ActivityRollupV1, CodeChangeMetric, ProjectInfo, - ProviderAccount, ProviderAccountId, QuotaCycleContributionV1, SourceAccountAssignment, - SourceAccountAssignmentId, SourceId, SourceLocation, Subscription, SubscriptionId, SummaryId, - TaskSpan, TaskVerification, TaskVerificationId, UsageEvent, UsageSummary, WorkItem, - WorkItemMember, + ProviderAccount, ProviderAccountId, QuotaCycleContributionV1, SessionRollupV1, + SourceAccountAssignment, SourceAccountAssignmentId, SourceId, SourceLocation, Subscription, + SubscriptionId, SummaryId, TaskSpan, TaskVerification, TaskVerificationId, UsageEvent, + UsageSummary, WorkItem, WorkItemMember, }; use chrono::{DateTime, Utc}; use schemars::JsonSchema; @@ -56,6 +56,10 @@ pub struct SyncBatch { /// Per-source coverage for activity kinds, including zero-call days. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub activity_coverage: Vec, + /// Opt-in session rollups. Empty batches omit the key. Each row is + /// `session_rollup.v1` with the closed ingest key set. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub sessions: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub authoritative_snapshot: Option, pub created_at: DateTime, @@ -88,6 +92,11 @@ pub struct SyncAuthoritativeSnapshot { pub activity_rollup_ids: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub activity_coverage_ids: Vec, + /// Present only when this collector is opting into session retirement. + /// An empty array prunes every hosted session for the device. Absence + /// leaves hosted sessions alone, including on pre-session v6 collectors. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub session_rollup_ids: Option>, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -145,6 +154,9 @@ pub struct SyncEntityCounts { pub activity_rollups: u64, #[serde(default, skip_serializing_if = "sync_count_is_zero")] pub activity_coverage: u64, + /// Present on `sync_ack.v6`. Earlier acknowledgements omit it. + #[serde(default, skip_serializing_if = "sync_count_is_zero")] + pub sessions: u64, } fn sync_count_is_zero(value: &u64) -> bool { @@ -209,3 +221,16 @@ pub fn sanitize_summary_for_sync(mut summary: UsageSummary) -> UsageSummary { } summary } + +/// Drops unstable projects and generic or secret-looking titles before a +/// session rollup leaves the device. `source_id` stays, matching summaries. +/// Nested usage and cost already serialize to the closed ingest keys. +#[must_use] +pub fn sanitize_session_rollup_for_sync(mut rollup: SessionRollupV1) -> SessionRollupV1 { + rollup.project = rollup.project.and_then(sanitize_project_for_sync); + if crate::task_title_is_generic(rollup.title.as_deref()) { + rollup.title = None; + rollup.title_source = None; + } + rollup +} diff --git a/crates/statsai-daemon/src/ingest.rs b/crates/statsai-daemon/src/ingest.rs index 3b3211b..0153668 100644 --- a/crates/statsai-daemon/src/ingest.rs +++ b/crates/statsai-daemon/src/ingest.rs @@ -65,6 +65,9 @@ pub fn ingest_sync_batch(store: &Store, batch: &SyncBatch) -> Result { { bail!("activity collections require sync_batch.v6"); } + if batch.schema_version != SYNC_BATCH_V6_SCHEMA_VERSION && !batch.sessions.is_empty() { + bail!("session rollups require sync_batch.v6"); + } if batch .code_change_metrics .iter() @@ -91,6 +94,9 @@ pub fn ingest_sync_batch(store: &Store, batch: &SyncBatch) -> Result { if !batch.activity_rollups.is_empty() || !batch.activity_coverage.is_empty() { bail!("activity collections are not supported by the loopback daemon"); } + if !batch.sessions.is_empty() { + bail!("session rollups are not supported by the loopback daemon"); + } let result = store.ingest_sync_batch(batch)?; @@ -112,6 +118,7 @@ pub fn ingest_sync_batch(store: &Store, batch: &SyncBatch) -> Result { account_evidence_summaries: batch.account_evidence_summaries.len() as u64, activity_rollups: 0, activity_coverage: 0, + sessions: 0, }, duplicates: SyncEntityCounts { sources: 0, @@ -129,6 +136,7 @@ pub fn ingest_sync_batch(store: &Store, batch: &SyncBatch) -> Result { account_evidence_summaries: 0, activity_rollups: 0, activity_coverage: 0, + sessions: 0, }, rejected: Vec::::new(), }) diff --git a/crates/statsai-daemon/src/tests.rs b/crates/statsai-daemon/src/tests.rs index 8df1c6d..adf4486 100644 --- a/crates/statsai-daemon/src/tests.rs +++ b/crates/statsai-daemon/src/tests.rs @@ -24,6 +24,7 @@ fn empty_batch() -> SyncBatch { account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: Vec::new(), @@ -329,6 +330,61 @@ fn ingest_v6_batch_refuses_activity_collections_it_cannot_store() { .contains("activity collections are not supported")); } +#[test] +fn ingest_refuses_session_rollups_the_loopback_daemon_cannot_store() { + let store = Store::in_memory().expect("store"); + let mut batch = empty_batch(); + batch.schema_version = SYNC_BATCH_V5_SCHEMA_VERSION.to_string(); + batch.sessions.push(loopback_session(&batch.device_id)); + let error = ingest_sync_batch(&store, &batch).expect_err("pre-v6 sessions"); + assert!(error + .to_string() + .contains("session rollups require sync_batch.v6")); + + batch.schema_version = SYNC_BATCH_V6_SCHEMA_VERSION.to_string(); + let error = ingest_sync_batch(&store, &batch).expect_err("unsupported sessions"); + assert!(error + .to_string() + .contains("session rollups are not supported")); +} + +fn loopback_session(device_id: &str) -> statsai_core::SessionRollupV1 { + let now = Utc::now(); + statsai_core::SessionRollupV1 { + schema_version: statsai_core::SESSION_ROLLUP_SCHEMA_VERSION.to_string(), + session_id: "session_loopback".to_string(), + device_id: device_id.to_string(), + provider: "codex".to_string(), + source_id: statsai_core::SourceId("source".to_string()), + provider_account_id: None, + started_at: now, + ended_at: now, + duration_seconds: Some(1), + usage: UsageCounts::default(), + requests: 1, + cost: statsai_core::CostInfo { + currency: "USD".to_string(), + estimated_api_equivalent_usd: None, + provider_reported_usd: None, + estimated_api_equivalent_micro_usd: None, + provider_reported_micro_usd: None, + pricing_source: None, + pricing_version: None, + confidence: Confidence::Medium, + }, + models: Vec::new(), + primary_model: None, + total_messages: None, + user_messages: None, + assistant_messages: None, + developer_messages: None, + project: None, + title: None, + title_source: None, + updated_at: now, + } +} + #[test] fn ingest_v3_batch_rejects_metric_owned_by_another_device_before_persisting() { let store = Store::in_memory().expect("store"); diff --git a/crates/statsai-store/src/accounts.rs b/crates/statsai-store/src/accounts.rs index 8a19e82..1071dbf 100644 --- a/crates/statsai-store/src/accounts.rs +++ b/crates/statsai-store/src/accounts.rs @@ -360,10 +360,17 @@ impl Store { .map(|value| parse_bool_metadata_value(SYNC_INCLUDE_ACTIVITY_METADATA_KEY, value)) .transpose()? .unwrap_or(false); + let include_sessions = self + .metadata_value(SYNC_INCLUDE_SESSIONS_METADATA_KEY)? + .as_deref() + .map(|value| parse_bool_metadata_value(SYNC_INCLUDE_SESSIONS_METADATA_KEY, value)) + .transpose()? + .unwrap_or(false); Ok(SyncPreferences { include_projects, include_tasks, include_activity, + include_sessions, } .normalized()) } @@ -390,6 +397,14 @@ impl Store { "0" }, )?; + self.set_metadata_value( + SYNC_INCLUDE_SESSIONS_METADATA_KEY, + if preferences.include_sessions { + "1" + } else { + "0" + }, + )?; Ok(()) } } diff --git a/crates/statsai-store/src/code_changes/tests/refresh.rs b/crates/statsai-store/src/code_changes/tests/refresh.rs index e3dddea..16af4fa 100644 --- a/crates/statsai-store/src/code_changes/tests/refresh.rs +++ b/crates/statsai-store/src/code_changes/tests/refresh.rs @@ -434,6 +434,7 @@ fn local_refresh_preserves_metrics_ingested_from_another_device() { account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: Vec::new(), diff --git a/crates/statsai-store/src/lib.rs b/crates/statsai-store/src/lib.rs index 63903e2..6c6ca78 100644 --- a/crates/statsai-store/src/lib.rs +++ b/crates/statsai-store/src/lib.rs @@ -114,6 +114,7 @@ const SYNC_ROLLUP_SUMMARY_VERSION: &str = "14"; const SYNC_INCLUDE_PROJECTS_METADATA_KEY: &str = "sync.include_projects"; const SYNC_INCLUDE_TASKS_METADATA_KEY: &str = "sync.include_tasks"; const SYNC_INCLUDE_ACTIVITY_METADATA_KEY: &str = "sync.include_activity"; +const SYNC_INCLUDE_SESSIONS_METADATA_KEY: &str = "sync.include_sessions"; const LEGACY_CODEX_PLAN_CONVERSION_METADATA_KEY: &str = "migration.legacy_codex_plan_evidence.v1"; const SQLITE_BUSY_TIMEOUT: Duration = if cfg!(test) { Duration::from_millis(50) @@ -193,17 +194,19 @@ pub struct SyncPreferences { pub include_projects: bool, pub include_tasks: bool, pub include_activity: bool, + pub include_sessions: bool, } impl SyncPreferences { #[must_use] pub fn normalized(self) -> Self { - let include_projects = self.include_projects || self.include_tasks; + let include_projects = self.include_projects || self.include_tasks || self.include_sessions; let include_tasks = self.include_tasks && include_projects; Self { include_projects, include_tasks, include_activity: self.include_activity, + include_sessions: self.include_sessions, } } } diff --git a/crates/statsai-store/src/rollups/sessions.rs b/crates/statsai-store/src/rollups/sessions.rs index 1db5c93..d22bf1c 100644 --- a/crates/statsai-store/src/rollups/sessions.rs +++ b/crates/statsai-store/src/rollups/sessions.rs @@ -192,6 +192,20 @@ impl Store { Ok(()) } + pub fn all_session_rollups(&self) -> Result> { + self.session_rollups_by_sql( + "SELECT payload FROM session_rollups ORDER BY started_at, session_id", + ) + } + + pub fn session_rollup_ids(&self) -> Result> { + let mut statement = self + .conn + .prepare("SELECT session_id FROM session_rollups ORDER BY session_id")?; + let rows = statement.query_map([], |row| row.get::<_, String>(0))?; + rows.collect::, _>>().map_err(Into::into) + } + pub fn dirty_session_rollups(&self) -> Result> { self.session_rollups_by_sql( "SELECT payload FROM session_rollups WHERE dirty = 1 ORDER BY started_at, session_id", diff --git a/crates/statsai-store/src/sync_state/mod.rs b/crates/statsai-store/src/sync_state/mod.rs index 11e8f86..9008f22 100644 --- a/crates/statsai-store/src/sync_state/mod.rs +++ b/crates/statsai-store/src/sync_state/mod.rs @@ -651,6 +651,20 @@ impl Store { .map(|coverage| coverage.coverage_id.clone()) .collect::>(), )?; + self.record_serialized_entities_synced_in_transaction( + sink, + target, + "session_rollup", + &batch.sessions, + |rollup| rollup.session_id.as_str(), + )?; + self.mark_session_rollups_synced_in_transaction( + &batch + .sessions + .iter() + .map(|rollup| rollup.session_id.clone()) + .collect::>(), + )?; self.record_task_bucket_snapshots_synced_in_transaction( sink, target, diff --git a/crates/statsai-store/src/sync_state/pending.rs b/crates/statsai-store/src/sync_state/pending.rs index 3aeb649..90b960d 100644 --- a/crates/statsai-store/src/sync_state/pending.rs +++ b/crates/statsai-store/src/sync_state/pending.rs @@ -162,6 +162,21 @@ impl Store { ) } + pub fn pending_session_rollups_for_sync( + &self, + sink: &str, + target: &str, + rollups: &[statsai_core::SessionRollupV1], + ) -> Result> { + self.pending_serialized_entities_for_sync( + sink, + target, + "session_rollup", + rollups, + |rollup| rollup.session_id.as_str(), + ) + } + pub fn pending_activity_coverage_for_sync( &self, sink: &str, @@ -331,6 +346,13 @@ impl Store { .collect::>(), ), ]); + let mut current_ids = current_ids; + if let Some(ids) = snapshot.session_rollup_ids.as_ref() { + current_ids.insert( + "session_rollup", + ids.iter().map(String::as_str).collect::>(), + ); + } let mut statement = self.conn.prepare( r#" SELECT entity_kind, entity_id @@ -340,7 +362,7 @@ impl Store { 'source', 'account', 'source_account_assignment', 'subscription', 'summary', 'code_change_metric', 'quota_cycle_contribution', 'account_plan_observation', 'account_evidence_summary', - 'activity_rollup', 'activity_coverage' + 'activity_rollup', 'activity_coverage', 'session_rollup' ) "#, )?; @@ -564,6 +586,11 @@ impl Store { .into_iter() .map(|coverage| coverage.coverage_id) .collect(), + session_rollup_ids: if self.sync_preferences()?.include_sessions { + Some(self.session_rollup_ids()?) + } else { + None + }, }) } } diff --git a/crates/statsai-store/src/sync_state/record.rs b/crates/statsai-store/src/sync_state/record.rs index e61a693..facf30b 100644 --- a/crates/statsai-store/src/sync_state/record.rs +++ b/crates/statsai-store/src/sync_state/record.rs @@ -314,6 +314,26 @@ impl Store { }) } + pub fn record_session_rollups_synced( + &self, + sink: &str, + target: &str, + rollups: &[statsai_core::SessionRollupV1], + ) -> Result<()> { + if rollups.is_empty() { + return Ok(()); + } + self.with_immediate_transaction(|| { + self.record_serialized_entities_synced_in_transaction( + sink, + target, + "session_rollup", + rollups, + |rollup| rollup.session_id.as_str(), + ) + }) + } + pub fn record_activity_coverage_synced( &self, sink: &str, diff --git a/crates/statsai-store/src/tests/sync_state/mod.rs b/crates/statsai-store/src/tests/sync_state/mod.rs index 073b309..ae1a399 100644 --- a/crates/statsai-store/src/tests/sync_state/mod.rs +++ b/crates/statsai-store/src/tests/sync_state/mod.rs @@ -146,6 +146,7 @@ fn record_rollup_chunk_sync_success_retries_busy_database() { account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: summaries.clone(), task_buckets: Vec::new(), @@ -400,6 +401,7 @@ fn sync_preferences_round_trip_and_normalize_tasks() { include_projects: false, include_tasks: true, include_activity: false, + include_sessions: false, }) .expect("save sync preferences"); @@ -409,6 +411,25 @@ fn sync_preferences_round_trip_and_normalize_tasks() { include_projects: true, include_tasks: true, include_activity: false, + include_sessions: false, + } + ); + + store + .set_sync_preferences(SyncPreferences { + include_projects: false, + include_tasks: false, + include_activity: false, + include_sessions: true, + }) + .expect("save session sync preferences"); + assert_eq!( + store.sync_preferences().expect("session sync preferences"), + SyncPreferences { + include_projects: true, + include_tasks: false, + include_activity: false, + include_sessions: true, } ); } diff --git a/crates/statsai-sync/src/lib.rs b/crates/statsai-sync/src/lib.rs index d844606..d29f0fc 100644 --- a/crates/statsai-sync/src/lib.rs +++ b/crates/statsai-sync/src/lib.rs @@ -336,6 +336,12 @@ fn validate_sync_ack(batch: &SyncBatch, ack: &SyncAck) -> Result<()> { ack.accepted.account_evidence_summaries, ack.duplicates.account_evidence_summaries, )?; + validate_sync_ack_counts( + "sessions", + batch.sessions.len() as u64, + ack.accepted.sessions, + ack.duplicates.sessions, + )?; Ok(()) } @@ -459,6 +465,7 @@ mod tests { account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: Vec::new(), diff --git a/crates/statsai/src/cli/args/sync.rs b/crates/statsai/src/cli/args/sync.rs index 24de0ce..6001a5e 100644 --- a/crates/statsai/src/cli/args/sync.rs +++ b/crates/statsai/src/cli/args/sync.rs @@ -55,7 +55,7 @@ pub(crate) struct SyncCommand { pub(crate) include_projects: bool, #[arg( long, - conflicts_with_all = ["include_projects", "include_tasks"], + conflicts_with_all = ["include_projects", "include_tasks", "include_sessions"], help = "Disable project metadata sync for this device and future syncs" )] pub(crate) exclude_projects: bool, @@ -83,4 +83,16 @@ pub(crate) struct SyncCommand { help = "Disable hosted agent activity sync and prune names on the next successful HTTP sync" )] pub(crate) exclude_activity: bool, + #[arg( + long, + conflicts_with_all = ["exclude_sessions", "exclude_projects"], + help = "Enable hosted session sync for this device and future syncs (implies --include-projects)" + )] + pub(crate) include_sessions: bool, + #[arg( + long, + conflicts_with = "include_sessions", + help = "Disable hosted session sync for this device and future syncs" + )] + pub(crate) exclude_sessions: bool, } diff --git a/crates/statsai/src/cli/sync/batch.rs b/crates/statsai/src/cli/sync/batch.rs index d3686d9..f8be41e 100644 --- a/crates/statsai/src/cli/sync/batch.rs +++ b/crates/statsai/src/cli/sync/batch.rs @@ -9,9 +9,10 @@ use statsai_core::{ }; use statsai_core::{ project_contains_file_paths, sanitize_code_change_metric_for_sync, - sanitize_task_bucket_for_sync, IdentitySource, ProjectInfo, ProviderAccount, - SourceAccountAssignment, SourceKind, SourceLocation, Subscription, SyncAuthoritativeSnapshot, - SyncBatch, TaskVerificationCursor, UsageEvent, UsageSummary, SYNC_BATCH_SCHEMA_VERSION, + sanitize_session_rollup_for_sync, sanitize_task_bucket_for_sync, IdentitySource, ProjectInfo, + ProviderAccount, SourceAccountAssignment, SourceKind, SourceLocation, Subscription, + SyncAuthoritativeSnapshot, SyncBatch, TaskVerificationCursor, UsageEvent, UsageSummary, + SYNC_BATCH_SCHEMA_VERSION, }; use statsai_store::{QuotaQuery, Store}; #[cfg(test)] @@ -277,6 +278,28 @@ pub(crate) fn build_sync_batch_with_identity_key( } else { all_activity_coverage.clone() }; + let all_sessions = if sync_preferences.include_sessions { + store + .all_session_rollups()? + .into_iter() + .map(|rollup| { + let mut rollup = sanitize_session_rollup_for_sync(rollup); + if !include_projects { + rollup.project = None; + } + rollup + }) + .collect::>() + } else { + Vec::new() + }; + let sessions = if !sync_preferences.include_sessions { + Vec::new() + } else if payload_mode == SyncPayloadMode::Rollups && !(command.full || state.is_none()) { + store.pending_session_rollups_for_sync(&command.sink, target, &all_sessions)? + } else { + all_sessions.clone() + }; if payload_mode == SyncPayloadMode::Rollups { let label = rollup_mode_label(command); @@ -327,6 +350,12 @@ pub(crate) fn build_sync_batch_with_identity_key( .iter() .map(|coverage| coverage.coverage_id.clone()) .collect(), + session_rollup_ids: sync_preferences.include_sessions.then(|| { + all_sessions + .iter() + .map(|rollup| rollup.session_id.clone()) + .collect() + }), }; let failed_without_resume = state.as_ref().is_some_and(|state| { state.failure_count > 0 && state.pending_resume_batch_id.is_none() @@ -403,6 +432,7 @@ pub(crate) fn build_sync_batch_with_identity_key( quota_cycle_contributions, activity_rollups, activity_coverage, + sessions, authoritative_snapshot, created_at, }, @@ -466,6 +496,13 @@ pub(crate) fn record_rollup_sync_chunk_success( .map(|coverage| coverage.coverage_id.clone()) .collect::>(), )?; + store.mark_session_rollups_synced( + &batch + .sessions + .iter() + .map(|rollup| rollup.session_id.clone()) + .collect::>(), + )?; snapshot::invalidate_dashboard_cache(); Ok(()) } @@ -516,6 +553,7 @@ pub(crate) fn record_sync_batch_success( // the next sync to that target resent every rollup. store.record_activity_rollups_synced(sink, target, &batch.activity_rollups)?; store.record_activity_coverage_synced(sink, target, &batch.activity_coverage)?; + store.record_session_rollups_synced(sink, target, &batch.sessions)?; store.mark_activity_rollups_synced( &batch .activity_rollups @@ -530,6 +568,13 @@ pub(crate) fn record_sync_batch_success( .map(|coverage| coverage.coverage_id.clone()) .collect::>(), )?; + store.mark_session_rollups_synced( + &batch + .sessions + .iter() + .map(|rollup| rollup.session_id.clone()) + .collect::>(), + )?; snapshot::invalidate_dashboard_cache(); Ok(()) } diff --git a/crates/statsai/src/cli/sync/chunking/budget.rs b/crates/statsai/src/cli/sync/chunking/budget.rs index dcfadf9..65ab38d 100644 --- a/crates/statsai/src/cli/sync/chunking/budget.rs +++ b/crates/statsai/src/cli/sync/chunking/budget.rs @@ -102,6 +102,18 @@ pub(crate) fn estimate_http_rollup_d1_queries(batch: &SyncBatch) -> usize { // ownership write, matching quota-cycle batching. let activity_queries = usize::from(!batch.activity_rollups.is_empty()) * 2 + usize::from(!batch.activity_coverage.is_empty()) * 2; + // Session rows are one lookup plus one write. A closing snapshot that + // names session_rollup_ids also deletes orphans the device no longer owns. + let session_queries = usize::from(!batch.sessions.is_empty()) * 2 + + usize::from( + batch + .authoritative_snapshot + .as_ref() + .is_some_and(|snapshot| { + snapshot.part_index.saturating_add(1) == snapshot.part_count + && snapshot.session_rollup_ids.is_some() + }), + ); let code_change_owner_metadata_refresh_queries = usize::from( batch.schema_version == SYNC_BATCH_SCHEMA_VERSION && batch @@ -127,6 +139,7 @@ pub(crate) fn estimate_http_rollup_d1_queries(batch: &SyncBatch) -> usize { + code_change_metric_queries + quota_cycle_contribution_queries + activity_queries + + session_queries + code_change_owner_metadata_refresh_queries + estimate_http_rollup_task_queries(batch) + final_sync_bookkeeping_queries diff --git a/crates/statsai/src/cli/sync/chunking/mod.rs b/crates/statsai/src/cli/sync/chunking/mod.rs index 98a3d01..57a1610 100644 --- a/crates/statsai/src/cli/sync/chunking/mod.rs +++ b/crates/statsai/src/cli/sync/chunking/mod.rs @@ -31,6 +31,7 @@ pub(crate) enum HttpRollupIndexedChunkKind { QuotaCycles, ActivityRollups, ActivityCoverage, + Sessions, Snapshot, } @@ -48,6 +49,7 @@ impl HttpRollupIndexedChunkKind { Self::QuotaCycles, Self::ActivityRollups, Self::ActivityCoverage, + Self::Sessions, Self::Snapshot, ]; @@ -74,6 +76,7 @@ impl HttpRollupIndexedChunkKind { Self::QuotaCycles => "quota_cycles", Self::ActivityRollups => "activity_rollups", Self::ActivityCoverage => "activity_coverage", + Self::Sessions => "sessions", Self::Snapshot => "snapshot", } } @@ -97,6 +100,8 @@ const HTTP_ROLLUP_QUOTA_CYCLE_CONTRIBUTIONS_PER_BATCH: usize = 100; pub(crate) const HTTP_ROLLUP_ACTIVITY_ROWS_PER_BATCH: usize = 500; +pub(crate) const HTTP_ROLLUP_SESSIONS_PER_BATCH: usize = 200; + pub(crate) const HTTP_ROLLUP_D1_QUERY_BUDGET: usize = 45; const HTTP_ROLLUP_D1_QUERY_CHUNK_SIZE: usize = 90; @@ -156,7 +161,9 @@ fn split_authoritative_snapshot( quota_cycle_contribution_ids: Vec::new(), activity_rollup_ids: Vec::new(), activity_coverage_ids: Vec::new(), + session_rollup_ids: None, }; + let session_rollup_ids = snapshot.session_rollup_ids; let mut parts = Vec::new(); let mut current = empty_part(); @@ -194,7 +201,27 @@ fn split_authoritative_snapshot( ); append_ids!(snapshot.activity_rollup_ids, activity_rollup_ids); append_ids!(snapshot.activity_coverage_ids, activity_coverage_ids); - if authoritative_snapshot_id_count(¤t) > 0 || parts.is_empty() { + if let Some(ids) = session_rollup_ids { + if ids.is_empty() { + // An empty array is the retirement signal. It adds no IDs, so it + // can ride on the current part even when that part is already full. + current.session_rollup_ids = Some(Vec::new()); + } else { + for id in ids { + if authoritative_snapshot_id_count(¤t) == max_ids { + parts.push(std::mem::replace(&mut current, empty_part())); + } + current + .session_rollup_ids + .get_or_insert_with(Vec::new) + .push(id); + } + } + } + if authoritative_snapshot_id_count(¤t) > 0 + || current.session_rollup_ids.is_some() + || parts.is_empty() + { parts.push(current); } let part_count = u32::try_from(parts.len()).expect("snapshot part count fits u32"); @@ -217,6 +244,7 @@ fn authoritative_snapshot_id_count(snapshot: &SyncAuthoritativeSnapshot) -> usiz + snapshot.quota_cycle_contribution_ids.len() + snapshot.activity_rollup_ids.len() + snapshot.activity_coverage_ids.len() + + snapshot.session_rollup_ids.as_ref().map_or(0, Vec::len) } pub(crate) fn split_http_rollup_sync_batches_without_snapshot(batch: &SyncBatch) -> Vec { @@ -232,13 +260,15 @@ pub(crate) fn split_http_rollup_sync_batches_without_snapshot(batch: &SyncBatch) || !batch.code_change_metrics.is_empty() || !batch.quota_cycle_contributions.is_empty() || !batch.activity_rollups.is_empty() - || !batch.activity_coverage.is_empty(); + || !batch.activity_coverage.is_empty() + || !batch.sessions.is_empty(); if !has_task_payload && batch.summaries.len() <= HTTP_ROLLUP_SUMMARIES_PER_BATCH && batch.code_change_metrics.len() <= HTTP_ROLLUP_CODE_CHANGE_METRICS_PER_BATCH && batch.quota_cycle_contributions.len() <= HTTP_ROLLUP_QUOTA_CYCLE_CONTRIBUTIONS_PER_BATCH && batch.activity_rollups.len() <= HTTP_ROLLUP_ACTIVITY_ROWS_PER_BATCH && batch.activity_coverage.len() <= HTTP_ROLLUP_ACTIVITY_ROWS_PER_BATCH + && sessions_fit_one_http_chunk(batch) && metadata_count <= HTTP_ROLLUP_METADATA_RECORDS_PER_BATCH { return fit_http_rollup_batches_to_d1_budget(vec![batch.clone()]); @@ -268,12 +298,17 @@ pub(crate) fn split_http_rollup_sync_batches_without_snapshot(batch: &SyncBatch) .activity_coverage .len() .div_ceil(HTTP_ROLLUP_ACTIVITY_ROWS_PER_BATCH); + let session_chunks = batch + .sessions + .len() + .div_ceil(HTTP_ROLLUP_SESSIONS_PER_BATCH); let mut chunks = Vec::with_capacity( total_chunks + metadata_chunks + code_change_chunks + quota_cycle_chunks + activity_chunks + + session_chunks + task_chunks.len(), ); @@ -294,6 +329,7 @@ pub(crate) fn split_http_rollup_sync_batches_without_snapshot(batch: &SyncBatch) batch, HTTP_ROLLUP_ACTIVITY_ROWS_PER_BATCH, )); + chunks.extend(split_http_session_chunks(batch)); chunks.extend(split_http_rollup_summary_chunks( batch, HTTP_ROLLUP_SUMMARIES_PER_BATCH, @@ -319,6 +355,14 @@ fn empty_http_rollup_chunk(batch: &SyncBatch, suffix: &str) -> SyncBatch { chunk.quota_cycle_contributions.clear(); chunk.activity_rollups.clear(); chunk.activity_coverage.clear(); + chunk.sessions.clear(); chunk.authoritative_snapshot = None; chunk } + +fn sessions_fit_one_http_chunk(batch: &SyncBatch) -> bool { + batch.sessions.len() <= HTTP_ROLLUP_SESSIONS_PER_BATCH + && serde_json::to_string(&batch.sessions) + .map(|payload| payload.len() <= TASK_SYNC_SQL_MAX_JSON_BYTES_PER_CHUNK) + .unwrap_or(false) +} diff --git a/crates/statsai/src/cli/sync/chunking/payloads.rs b/crates/statsai/src/cli/sync/chunking/payloads.rs index b2d7930..0fe95a9 100644 --- a/crates/statsai/src/cli/sync/chunking/payloads.rs +++ b/crates/statsai/src/cli/sync/chunking/payloads.rs @@ -1,6 +1,18 @@ use super::*; pub(crate) fn split_http_rollup_sync_batch_after_budget_error(batch: &SyncBatch) -> Vec { + if !batch.sessions.is_empty() && has_non_session_payload(batch) { + let mut without_sessions = batch.clone(); + without_sessions.sessions.clear(); + let mut chunks = split_http_session_chunks(batch); + chunks.extend(split_http_rollup_sync_batch_after_budget_error( + &without_sessions, + )); + return chunks; + } + if batch.sessions.len() > 1 { + return split_http_session_chunks_halved(batch); + } if (!batch.activity_rollups.is_empty() || !batch.activity_coverage.is_empty()) && has_non_activity_payload(batch) { @@ -143,6 +155,7 @@ pub(crate) fn has_non_code_change_payload(batch: &SyncBatch) -> bool { || !batch.quota_cycle_contributions.is_empty() || !batch.activity_rollups.is_empty() || !batch.activity_coverage.is_empty() + || !batch.sessions.is_empty() } pub(crate) fn has_non_quota_cycle_payload(batch: &SyncBatch) -> bool { @@ -154,6 +167,19 @@ pub(crate) fn has_non_quota_cycle_payload(batch: &SyncBatch) -> bool { || !batch.code_change_metrics.is_empty() || !batch.activity_rollups.is_empty() || !batch.activity_coverage.is_empty() + || !batch.sessions.is_empty() +} + +pub(crate) fn has_non_session_payload(batch: &SyncBatch) -> bool { + http_rollup_metadata_count(batch) > 0 + || !batch.events.is_empty() + || !batch.summaries.is_empty() + || !batch.task_buckets.is_empty() + || !batch.task_verifications.is_empty() + || !batch.code_change_metrics.is_empty() + || !batch.quota_cycle_contributions.is_empty() + || !batch.activity_rollups.is_empty() + || !batch.activity_coverage.is_empty() } pub(crate) fn has_non_activity_payload(batch: &SyncBatch) -> bool { @@ -164,6 +190,7 @@ pub(crate) fn has_non_activity_payload(batch: &SyncBatch) -> bool { || !batch.task_verifications.is_empty() || !batch.code_change_metrics.is_empty() || !batch.quota_cycle_contributions.is_empty() + || !batch.sessions.is_empty() } /// Records the backend writes one statement per row for. @@ -260,6 +287,62 @@ pub(crate) fn split_http_quota_cycle_contribution_chunks( .collect() } +pub(crate) fn split_http_session_chunks(batch: &SyncBatch) -> Vec { + let mut chunks = Vec::new(); + let mut current = Vec::new(); + let mut current_bytes = 2usize; + for session in &batch.sessions { + let row_bytes = serde_json::to_string(session) + .map(|row| row.len()) + .unwrap_or(0); + let next_bytes = if current.is_empty() { + 2 + row_bytes + } else { + current_bytes + 1 + row_bytes + }; + if !current.is_empty() + && (current.len() >= HTTP_ROLLUP_SESSIONS_PER_BATCH + || next_bytes > TASK_SYNC_SQL_MAX_JSON_BYTES_PER_CHUNK) + { + chunks.push(session_chunk(batch, chunks.len(), ¤t)); + current.clear(); + current_bytes = 2; + } + current.push(session.clone()); + current_bytes = if current.len() == 1 { + 2 + row_bytes + } else { + current_bytes + 1 + row_bytes + }; + } + if !current.is_empty() { + chunks.push(session_chunk(batch, chunks.len(), ¤t)); + } + chunks +} + +fn split_http_session_chunks_halved(batch: &SyncBatch) -> Vec { + let midpoint = batch.sessions.len().div_ceil(2).max(1); + let mut chunks = Vec::new(); + for (index, sessions) in batch.sessions.chunks(midpoint).enumerate() { + chunks.push(session_chunk(batch, index, sessions)); + } + chunks +} + +fn session_chunk( + batch: &SyncBatch, + index: usize, + sessions: &[statsai_core::SessionRollupV1], +) -> SyncBatch { + let mut chunk = empty_http_rollup_chunk( + batch, + &HttpRollupIndexedChunkKind::Sessions.suffix(index + 1), + ); + chunk.sessions = sessions.to_vec(); + chunk +} + pub(crate) fn split_http_activity_chunks(batch: &SyncBatch, chunk_size: usize) -> Vec { let chunk_size = chunk_size.max(1); let mut chunks = Vec::new(); @@ -376,6 +459,7 @@ pub(crate) fn split_http_rollup_single_metadata_kind( | HttpRollupIndexedChunkKind::QuotaCycles | HttpRollupIndexedChunkKind::ActivityRollups | HttpRollupIndexedChunkKind::ActivityCoverage + | HttpRollupIndexedChunkKind::Sessions | HttpRollupIndexedChunkKind::Snapshot => Vec::new(), } } @@ -411,6 +495,7 @@ pub(crate) fn split_http_rollup_summary_chunks( chunk.quota_cycle_contributions.clear(); chunk.activity_rollups.clear(); chunk.activity_coverage.clear(); + chunk.sessions.clear(); chunk.authoritative_snapshot = None; chunk }) diff --git a/crates/statsai/src/cli/sync/mod.rs b/crates/statsai/src/cli/sync/mod.rs index 9d5bba4..81837c7 100644 --- a/crates/statsai/src/cli/sync/mod.rs +++ b/crates/statsai/src/cli/sync/mod.rs @@ -36,6 +36,7 @@ pub(crate) fn effective_sync_preferences( if command.exclude_projects { preferences.include_projects = false; preferences.include_tasks = false; + preferences.include_sessions = false; } if command.include_tasks { preferences.include_projects = true; @@ -50,6 +51,13 @@ pub(crate) fn effective_sync_preferences( if command.exclude_activity { preferences.include_activity = false; } + if command.include_sessions { + preferences.include_projects = true; + preferences.include_sessions = true; + } + if command.exclude_sessions { + preferences.include_sessions = false; + } Ok(preferences.normalized()) } @@ -63,7 +71,7 @@ fn apply_sync_preference_overrides( if preferences != original { store.set_sync_preferences(preferences)?; eprintln!( - "sync preferences updated: projects={} tasks={} activity={}", + "sync preferences updated: projects={} tasks={} activity={} sessions={}", if preferences.include_projects { "enabled" } else { @@ -78,11 +86,17 @@ fn apply_sync_preference_overrides( "enabled" } else { "disabled" + }, + if preferences.include_sessions { + "enabled" + } else { + "disabled" } ); if (!original.include_projects && preferences.include_projects) || (!original.include_tasks && preferences.include_tasks) || (!original.include_activity && preferences.include_activity) + || (!original.include_sessions && preferences.include_sessions) { eprintln!( "sync preferences changed privacy/backfill scope; the next sync may resend historical summaries to update the hosted mirror" @@ -101,6 +115,16 @@ fn apply_sync_preference_overrides( "activity sync disabled: the next successful HTTP sync sends an empty activity snapshot so hosted names are pruned" ); } + if !original.include_sessions && preferences.include_sessions { + eprintln!( + "session sync enabled: the next sync backfills session rollups, and later snapshots retire hosted sessions this device no longer has" + ); + } + if original.include_sessions && !preferences.include_sessions { + eprintln!( + "session sync disabled: hosted sessions stay in place until session sync is enabled again" + ); + } } Ok(preferences) } @@ -174,20 +198,21 @@ pub(crate) fn sync(command: SyncCommand, store: &Store, device_id: &str) -> Resu if command.dry_run { eprintln!( - "dry run: sink={} mode={} include_projects={} include_tasks={} include_activity={} sources={} events={} summaries={} task_buckets={} task_verifications={} activity_rollups={} activity_coverage={}", + "dry run: sink={} mode={} include_projects={} include_tasks={} include_activity={} include_sessions={} sources={} events={} summaries={} task_buckets={} task_verifications={} activity_rollups={} activity_coverage={} sessions={}", command.sink, sync_payload_mode_name(payload_mode), sync_preferences.include_projects, sync_preferences.include_tasks, sync_preferences.include_activity, + sync_preferences.include_sessions, batch.sources.len(), batch.events.len(), - batch.summaries.len() - , + batch.summaries.len(), batch.task_buckets.len(), batch.task_verifications.len(), batch.activity_rollups.len(), batch.activity_coverage.len(), + batch.sessions.len(), ); return Ok(()); } @@ -903,6 +928,8 @@ mod tests { exclude_tasks: false, include_activity: false, exclude_activity: false, + include_sessions: false, + exclude_sessions: false, } } diff --git a/crates/statsai/src/cli/tests/support.rs b/crates/statsai/src/cli/tests/support.rs index f084db6..8200dde 100644 --- a/crates/statsai/src/cli/tests/support.rs +++ b/crates/statsai/src/cli/tests/support.rs @@ -106,6 +106,8 @@ pub(super) fn test_sync_command(sink: &str) -> SyncCommand { exclude_tasks: false, include_activity: false, exclude_activity: false, + include_sessions: false, + exclude_sessions: false, } } diff --git a/crates/statsai/src/cli/tests/sync/batch.rs b/crates/statsai/src/cli/tests/sync/batch.rs index 5d51eff..5905fd5 100644 --- a/crates/statsai/src/cli/tests/sync/batch.rs +++ b/crates/statsai/src/cli/tests/sync/batch.rs @@ -1508,6 +1508,7 @@ fn sync_batch_serialization_excludes_local_task_entities() { account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: Vec::new(), @@ -1612,6 +1613,7 @@ fn build_sync_batch_respects_project_and_task_opt_ins() { include_projects: true, include_tasks: false, include_activity: false, + include_sessions: false, }) .expect("persist sync preferences"); let (persisted_batch, persisted_mode) = @@ -1703,6 +1705,7 @@ fn code_change_metric_project_ids_follow_sync_project_preferences() { include_projects: true, include_tasks: false, include_activity: false, + include_sessions: false, }) .expect("persist project opt-in"); let exclude_command = SyncCommand { @@ -2094,6 +2097,7 @@ fn activity_sync_payload_omits_paths_commands_and_invocation_ids() { let include_command = SyncCommand { include_activity: true, + include_sessions: false, ..test_sync_command("file") }; let include_target = sync_target(&include_command).expect("target"); diff --git a/crates/statsai/src/cli/tests/sync/chunking/budget.rs b/crates/statsai/src/cli/tests/sync/chunking/budget.rs index 21e008a..65aa95c 100644 --- a/crates/statsai/src/cli/tests/sync/chunking/budget.rs +++ b/crates/statsai/src/cli/tests/sync/chunking/budget.rs @@ -48,6 +48,7 @@ fn http_rollup_sync_proactively_splits_batches_to_fit_d1_budget() { account_evidence_summaries: vec![], activity_rollups: vec![], activity_coverage: vec![], + sessions: vec![], events: vec![], summaries, task_buckets: vec![], @@ -201,6 +202,7 @@ fn http_rollup_project_counts_include_path_only_projects() { account_evidence_summaries: vec![], activity_rollups: vec![], activity_coverage: vec![], + sessions: vec![], events: vec![], summaries: vec![summary], task_buckets: vec![], @@ -304,6 +306,7 @@ fn test_dense_task_only_sync_batch(now: DateTime, span_count: usize) -> Syn account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: vec![TaskBucketSnapshot { @@ -505,6 +508,7 @@ fn test_multi_bucket_dense_task_only_sync_batch( account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets, diff --git a/crates/statsai/src/cli/tests/sync/chunking/mod.rs b/crates/statsai/src/cli/tests/sync/chunking/mod.rs index a1bd939..e32ddec 100644 --- a/crates/statsai/src/cli/tests/sync/chunking/mod.rs +++ b/crates/statsai/src/cli/tests/sync/chunking/mod.rs @@ -41,6 +41,7 @@ fn http_rollup_sync_splits_large_summary_batches() { account_evidence_summaries: vec![], activity_rollups: vec![], activity_coverage: vec![], + sessions: vec![], events: vec![], summaries, task_buckets: vec![], @@ -94,6 +95,7 @@ fn http_rollup_sync_sends_authoritative_snapshot_after_data_chunks() { account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: Vec::new(), @@ -150,6 +152,7 @@ fn http_rollup_sync_bounds_authoritative_snapshot_chunks() { account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: Vec::new(), @@ -274,6 +277,7 @@ fn http_rollup_sync_splits_metadata_away_from_summaries() { account_evidence_summaries: vec![], activity_rollups: vec![], activity_coverage: vec![], + sessions: vec![], events: vec![], summaries, task_buckets: vec![], @@ -346,6 +350,7 @@ fn http_rollup_metadata_budget_retries_preserve_all_metadata_kinds() { account_evidence_summaries: vec![], activity_rollups: vec![], activity_coverage: vec![], + sessions: vec![], events: vec![], summaries: vec![], task_buckets: vec![], @@ -382,3 +387,69 @@ fn http_rollup_metadata_budget_retries_preserve_all_metadata_kinds() { assert!(chunks.iter().any(|chunk| !chunk.sources.is_empty())); assert!(chunks.iter().any(|chunk| !chunk.accounts.is_empty())); } + +#[test] +fn http_rollup_snapshot_preserves_opt_in_session_retirement() { + let now = Utc + .with_ymd_and_hms(2026, 5, 29, 10, 12, 43) + .single() + .expect("date"); + let empty = SyncBatch { + schema_version: SYNC_BATCH_SCHEMA_VERSION.to_string(), + batch_id: "batch_empty_sessions".to_string(), + device_id: "device".to_string(), + sources: Vec::new(), + accounts: Vec::new(), + source_account_assignments: Vec::new(), + subscriptions: Vec::new(), + account_plan_observations: Vec::new(), + account_evidence_summaries: Vec::new(), + activity_rollups: Vec::new(), + activity_coverage: Vec::new(), + sessions: Vec::new(), + events: Vec::new(), + summaries: Vec::new(), + task_buckets: Vec::new(), + task_verifications: Vec::new(), + code_change_metrics: Vec::new(), + quota_cycle_contributions: Vec::new(), + authoritative_snapshot: Some(SyncAuthoritativeSnapshot { + session_rollup_ids: Some(Vec::new()), + ..SyncAuthoritativeSnapshot::default() + }), + created_at: now, + }; + let empty_snapshot = split_http_rollup_sync_batches(&empty) + .into_iter() + .find_map(|chunk| chunk.authoritative_snapshot) + .expect("snapshot chunk"); + assert_eq!(empty_snapshot.session_rollup_ids.as_deref(), Some(&[][..])); + let json = serde_json::to_value(&empty_snapshot).expect("serialize"); + assert_eq!(json["session_rollup_ids"], serde_json::json!([])); + + let ids = (0..=HTTP_ROLLUP_SNAPSHOT_IDS_PER_BATCH) + .map(|index| format!("session-{index}")) + .collect::>(); + let split = SyncBatch { + batch_id: "batch_split_sessions".to_string(), + authoritative_snapshot: Some(SyncAuthoritativeSnapshot { + session_rollup_ids: Some(ids.clone()), + ..SyncAuthoritativeSnapshot::default() + }), + ..empty + }; + let parts = split_http_rollup_sync_batches(&split) + .into_iter() + .filter_map(|chunk| chunk.authoritative_snapshot) + .collect::>(); + assert!(parts.len() > 1); + let seen = parts + .iter() + .flat_map(|part| { + part.session_rollup_ids + .clone() + .expect("parts that carry session ids keep the retirement key") + }) + .collect::>(); + assert_eq!(seen, ids); +} diff --git a/crates/statsai/src/cli/tests/sync/chunking/payloads.rs b/crates/statsai/src/cli/tests/sync/chunking/payloads.rs index 89416ea..ccfe0ec 100644 --- a/crates/statsai/src/cli/tests/sync/chunking/payloads.rs +++ b/crates/statsai/src/cli/tests/sync/chunking/payloads.rs @@ -37,6 +37,7 @@ fn test_quota_only_sync_batch(now: DateTime, count: usize) -> SyncBatch { account_evidence_summaries: vec![], activity_rollups: vec![], activity_coverage: vec![], + sessions: vec![], events: vec![], summaries: vec![], task_buckets: vec![], @@ -264,6 +265,7 @@ fn http_rollup_sends_metadata_before_task_chunks() { account_evidence_summaries: vec![], activity_rollups: vec![], activity_coverage: vec![], + sessions: vec![], events: vec![], summaries: vec![], task_buckets: test_task_only_sync_batch(now, 1, 0).task_buckets, diff --git a/crates/statsai/src/cli/tests/sync/http.rs b/crates/statsai/src/cli/tests/sync/http.rs index d6d1d7a..e51c4d3 100644 --- a/crates/statsai/src/cli/tests/sync/http.rs +++ b/crates/statsai/src/cli/tests/sync/http.rs @@ -55,6 +55,7 @@ fn http_rollup_sync_retries_smaller_batches_after_budget_rejection() { account_evidence_summaries: vec![], activity_rollups: vec![], activity_coverage: vec![], + sessions: vec![], events: vec![], summaries, task_buckets: vec![], @@ -152,6 +153,7 @@ fn http_rollup_sync_retries_smaller_batches_after_payload_too_large() { account_evidence_summaries: vec![], activity_rollups: vec![], activity_coverage: vec![], + sessions: vec![], events: vec![], summaries, task_buckets: vec![], diff --git a/crates/statsai/src/cli/tests/sync/mod.rs b/crates/statsai/src/cli/tests/sync/mod.rs index d29e048..174ef7d 100644 --- a/crates/statsai/src/cli/tests/sync/mod.rs +++ b/crates/statsai/src/cli/tests/sync/mod.rs @@ -308,6 +308,7 @@ pub(crate) fn test_task_only_sync_batch( account_evidence_summaries: Vec::new(), activity_rollups: Vec::new(), activity_coverage: Vec::new(), + sessions: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets, diff --git a/crates/statsai/tests/cli_surface/help.txt b/crates/statsai/tests/cli_surface/help.txt index 09e46a2..4a29a61 100644 --- a/crates/statsai/tests/cli_surface/help.txt +++ b/crates/statsai/tests/cli_surface/help.txt @@ -1058,6 +1058,8 @@ Options: --exclude-tasks Disable hosted task sync for this device and future syncs --include-activity Enable hosted agent activity sync for this device and future syncs --exclude-activity Disable hosted agent activity sync and prune names on the next successful HTTP sync + --include-sessions Enable hosted session sync for this device and future syncs (implies --include-projects) + --exclude-sessions Disable hosted session sync for this device and future syncs -h, --help Print help ===== statsai schema --help ===== Print JSON schemas for backend-facing contracts diff --git a/crates/statsai/tests/cli_surface/schema-sync-batch.json b/crates/statsai/tests/cli_surface/schema-sync-batch.json index 19888d4..f9fb267 100644 --- a/crates/statsai/tests/cli_surface/schema-sync-batch.json +++ b/crates/statsai/tests/cli_surface/schema-sync-batch.json @@ -86,6 +86,13 @@ "schema_version": { "type": "string" }, + "sessions": { + "description": "Opt-in session rollups. Empty batches omit the key. Each row is `session_rollup.v1` with the closed ingest key set.", + "type": "array", + "items": { + "$ref": "#/definitions/SessionRollupV1" + } + }, "source_account_assignments": { "type": "array", "items": { @@ -1421,6 +1428,40 @@ } } }, + "SessionCostWire": { + "description": "Cost keys accepted on `session_rollup.v1`.", + "type": "object", + "properties": { + "estimated_api_equivalent_micro_usd": { + "type": [ + "integer", + "null" + ], + "format": "int64" + }, + "estimated_api_equivalent_usd": { + "type": [ + "integer", + "null" + ], + "format": "int64" + }, + "provider_reported_micro_usd": { + "type": [ + "integer", + "null" + ], + "format": "int64" + }, + "provider_reported_usd": { + "type": [ + "integer", + "null" + ], + "format": "int64" + } + } + }, "SessionInfo": { "type": "object", "required": [ @@ -1464,6 +1505,274 @@ } } }, + "SessionModelWire": { + "description": "One model row on `session_rollup.v1`.", + "type": "object", + "required": [ + "cost", + "model", + "usage" + ], + "properties": { + "cost": { + "$ref": "#/definitions/SessionCostWire" + }, + "metrics": { + "anyOf": [ + { + "$ref": "#/definitions/SummaryModelMetrics" + }, + { + "type": "null" + } + ] + }, + "model": { + "$ref": "#/definitions/ModelInfo" + }, + "usage": { + "$ref": "#/definitions/SessionUsageWire" + } + } + }, + "SessionRollupV1": { + "description": "One local session, aggregated from the usage events that share its hashed id.\n\n`session_id` is the hashed `session_…` identity already stored on each event. Raw provider session ids never appear on this record.", + "type": "object", + "required": [ + "cost", + "device_id", + "duration_seconds", + "ended_at", + "models", + "provider", + "requests", + "schema_version", + "session_id", + "source_id", + "started_at", + "updated_at", + "usage" + ], + "properties": { + "assistant_messages": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "cost": { + "description": "Closed cost object. Currency, pricing source, and confidence stay local.", + "allOf": [ + { + "$ref": "#/definitions/SessionCostWire" + } + ] + }, + "developer_messages": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "device_id": { + "type": "string" + }, + "duration_seconds": { + "description": "Always a non-negative integer on the wire. Unknown durations serialize as 0 because `session_rollup.v1` rejects null.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "ended_at": { + "type": "string", + "format": "date-time" + }, + "models": { + "type": "array", + "items": { + "$ref": "#/definitions/SessionModelWire" + } + }, + "primary_model": { + "type": [ + "string", + "null" + ] + }, + "project": { + "anyOf": [ + { + "$ref": "#/definitions/ProjectInfo" + }, + { + "type": "null" + } + ] + }, + "provider": { + "type": "string" + }, + "provider_account_id": { + "type": [ + "string", + "null" + ] + }, + "requests": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "schema_version": { + "type": "string" + }, + "session_id": { + "type": "string" + }, + "source_id": { + "type": "string" + }, + "started_at": { + "type": "string", + "format": "date-time" + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "title_source": { + "anyOf": [ + { + "$ref": "#/definitions/SessionTitleSource" + }, + { + "type": "null" + } + ] + }, + "total_messages": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "updated_at": { + "type": "string", + "format": "date-time" + }, + "usage": { + "description": "Closed usage object. Cache-lifetime splits stay local; the ingest allowlist does not include them.", + "allOf": [ + { + "$ref": "#/definitions/SessionUsageWire" + } + ] + }, + "user_messages": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + } + } + }, + "SessionTitleSource": { + "description": "Where a session title was resolved from. Event titles win over task spans, which win over archived conversations.", + "type": "string", + "enum": [ + "event", + "task_span", + "archive" + ] + }, + "SessionUsageWire": { + "description": "Usage keys accepted on `session_rollup.v1`. Five-minute and one-hour cache splits are folded into `cache_creation_tokens` before this leaves the device.", + "type": "object", + "properties": { + "cache_creation_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "cache_read_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "input_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "local_eval_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "local_prompt_eval_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "output_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "reasoning_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "requests": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "total_tokens": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + } + } + }, "SkillCatalog": { "type": "string", "enum": [ @@ -2055,6 +2364,16 @@ "type": "string" } }, + "session_rollup_ids": { + "description": "Present only when this collector is opting into session retirement. An empty array prunes every hosted session for the device. Absence leaves hosted sessions alone, including on pre-session v6 collectors.", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + } + }, "snapshot_id": { "type": "string" }, diff --git a/docs/sync-contract.md b/docs/sync-contract.md index 4662c13..7d8fd2e 100644 --- a/docs/sync-contract.md +++ b/docs/sync-contract.md @@ -23,6 +23,16 @@ names leave the device only when the collector preference `include_activity` is on; turning it off sends an authoritative empty activity ID set so the backend prunes hosted names. Invocation IDs, arguments, outputs, commands, prompts, and file paths never appear in a v6 payload. +The same batch schema carries opt-in `sessions` rows. Each row is +`session_rollup.v1` with a closed key set: hashed session id, provider, +source, account, time range, duration, usage, requests, cost, models, message +counts, project, bounded title, and `title_source` (`event`, `task_span`, or +`archive`). Prompts, responses, paths, and raw provider session ids stay on +the device. Session retirement is not implied by `sync_batch.v6`. A snapshot +retires hosted sessions only when it includes `session_rollup_ids`. An empty +array prunes them. Omitting the key leaves hosted sessions in place, which is +what the collector does when `include_sessions` is off. Pre-v6 acknowledgements +omit the `sessions` counter. The collector owns local scanning, normalization, idempotent local storage, and privacy scrubbing. The backend owns authentication, validation, deduplication, rollups, and user-facing queries. The production path sends sanitized batches to @@ -273,7 +283,8 @@ observations and derives its own cycles from them, so acknowledging another device's cycles would tell the sender they had been stored when they had not. It likewise rejects `activity_rollups` and `activity_coverage`: hosted activity is a Cloudflare-only collection, and the loopback daemon has nowhere to store -another device's names. +another device's names. It rejects `sessions` for the same reason: hosted +session rollups are stored only by `/api/sync/batches`. `/api/sync/batches` is the production contract. A compatible backend should: - require an authenticated device access token @@ -289,7 +300,8 @@ another device's names. device; v3 fragments also carry code-change metric IDs; v4 fragments also carry quota-cycle contribution IDs; v5 fragments also carry account-plan observation and evidence-summary IDs; v6 fragments also carry activity - rollup and coverage IDs; each fragment carries zero-based `part_index` and a + rollup and coverage IDs and, when the collector opts into session retirement, + `session_rollup_ids`; each fragment carries zero-based `part_index` and a common `part_count`, with at most 200 IDs across its ID arrays - stage snapshot ownership without pruning until the final in-order fragment; then apply ownership and deletion reconciliation atomically, pruning a hosted @@ -359,7 +371,8 @@ run immediately. `sync_batch.v5` returns `sync_ack.v5`, which additionally adds the `account_plan_observations` and `account_evidence_summaries` counters. `sync_batch.v6` returns `sync_ack.v6`, which additionally adds the -`activity_rollups` and `activity_coverage` counters. +`activity_rollups`, `activity_coverage`, and `sessions` counters. Earlier +acknowledgements omit `sessions`. Collectors require the acknowledgement version to match the submitted batch version exactly; a v1 acknowledgement cannot successfully acknowledge a v2 batch, a v2 acknowledgement cannot acknowledge a v3 batch, a v3 @@ -489,7 +502,7 @@ verification actions for `sync_batch.v2` and later, plus code-change metrics for `sync_batch.v3` and later, plus quota-cycle contributions for `sync_batch.v4` and later, plus privacy-safe account-plan evidence for `sync_batch.v5`, plus opt-in activity rollups and coverage for -`sync_batch.v6`. The collector now prepares those +`sync_batch.v6`, plus opt-in session rollups on that same v6 schema. The collector now prepares those daily rollups before HTTP sync, so a normal Cloudflare sync can populate the dashboard without shipping raw events. Repeated batches are idempotent by stable IDs. From cab2605d2932247eb9ff25bd02ea8be0dffab95e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 25 Sep 2026 19:54:27 +0000 Subject: [PATCH 03/16] Keep session sync payloads inside the ingest limits Repair oversized titles and totals the API rejects so one session cannot fail the batch and block snapshot retirement. Order inverted session intervals so duration stays non-negative. Co-authored-by: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> --- crates/statsai-core/src/tests/types.rs | 270 +++++++++++++++++ crates/statsai-core/src/types/sync.rs | 277 +++++++++++++++++- crates/statsai-store/src/rollups/sessions.rs | 7 +- .../src/tests/rollups/sessions.rs | 39 +++ 4 files changed, 587 insertions(+), 6 deletions(-) diff --git a/crates/statsai-core/src/tests/types.rs b/crates/statsai-core/src/tests/types.rs index 1bc3731..d41b1e3 100644 --- a/crates/statsai-core/src/tests/types.rs +++ b/crates/statsai-core/src/tests/types.rs @@ -608,3 +608,273 @@ fn sanitize_summary_for_sync_marks_project_path_labels_as_file_paths() { ); assert!(sanitized.privacy.contains_file_paths); } + +fn sample_session_rollup(started: DateTime) -> SessionRollupV1 { + SessionRollupV1 { + schema_version: SESSION_ROLLUP_SCHEMA_VERSION.to_string(), + session_id: "session_abc".to_string(), + device_id: "device".to_string(), + provider: "codex".to_string(), + source_id: SourceId("src".to_string()), + provider_account_id: None, + started_at: started, + ended_at: started, + duration_seconds: Some(0), + usage: UsageCounts::default(), + requests: 1, + cost: CostInfo { + currency: "USD".to_string(), + estimated_api_equivalent_usd: None, + provider_reported_usd: None, + estimated_api_equivalent_micro_usd: Some(10), + provider_reported_micro_usd: None, + pricing_source: None, + pricing_version: None, + confidence: Confidence::Medium, + }, + models: Vec::new(), + primary_model: Some("gpt-5".to_string()), + total_messages: None, + user_messages: None, + assistant_messages: None, + developer_messages: None, + project: None, + title: Some("Fix parser".to_string()), + title_source: Some(SessionTitleSource::TaskSpan), + updated_at: started, + } +} + +#[test] +fn sanitize_session_rollup_repairs_values_ingest_rejects() { + let started = DateTime::parse_from_rfc3339("2026-05-01T00:00:00Z") + .expect("start") + .with_timezone(&Utc); + let mut inverted = sample_session_rollup(started + chrono::Duration::hours(2)); + inverted.ended_at = started; + inverted.duration_seconds = None; + let sanitized = sanitize_session_rollup_for_sync(inverted); + assert_eq!(sanitized.ended_at, sanitized.started_at); + assert_eq!(sanitized.duration_seconds, Some(0)); + + let mut future = sample_session_rollup(started); + future.started_at = Utc::now() + chrono::Duration::hours(48); + future.ended_at = future.started_at; + let sanitized = sanitize_session_rollup_for_sync(future); + assert_eq!(sanitized.started_at, started); + assert_eq!(sanitized.ended_at, started); + assert!(sanitized.updated_at <= Utc::now() + chrono::Duration::hours(24)); + + let mut titled = sample_session_rollup(started); + let mut title = "A".repeat(300); + title.insert(10, '\n'); + titled.title = Some(title); + titled.title_source = Some(SessionTitleSource::Event); + let sanitized = sanitize_session_rollup_for_sync(titled); + let title = sanitized.title.expect("repaired title"); + assert!(!title.chars().any(|ch| ch.is_control())); + assert!(title.encode_utf16().count() <= 256); + assert_eq!(sanitized.title_source, Some(SessionTitleSource::Event)); + + let mut generic = sample_session_rollup(started); + generic.title = Some("work item".to_string()); + generic.title_source = Some(SessionTitleSource::Archive); + let sanitized = sanitize_session_rollup_for_sync(generic); + assert!(sanitized.title.is_none()); + assert!(sanitized.title_source.is_none()); + + let mut counted = sample_session_rollup(started); + counted.usage.input_tokens = Some(1_000_000_000_005); + counted.usage.requests = Some(1_000_000_005); + counted.requests = 1_000_000_005; + counted.cost.estimated_api_equivalent_micro_usd = Some(-25); + counted.total_messages = Some(u64::MAX); + counted.models = (0..65) + .map(|index| SummaryModelUsage { + model: ModelInfo { + normalized_name: Some(format!("m{index}")), + ..ModelInfo::default() + }, + usage: UsageCounts { + total_tokens: Some(index), + ..UsageCounts::default() + }, + cost: CostInfo { + currency: "USD".to_string(), + estimated_api_equivalent_usd: None, + provider_reported_usd: None, + estimated_api_equivalent_micro_usd: None, + provider_reported_micro_usd: None, + pricing_source: None, + pricing_version: None, + confidence: Confidence::Medium, + }, + metrics: None, + }) + .collect(); + let sanitized = sanitize_session_rollup_for_sync(counted); + assert_eq!(sanitized.usage.input_tokens, Some(1_000_000_000_000)); + assert_eq!(sanitized.usage.requests, Some(1_000_000_000)); + assert_eq!(sanitized.requests, 1_000_000_000); + assert_eq!(sanitized.cost.estimated_api_equivalent_micro_usd, Some(0)); + assert_eq!(sanitized.total_messages, Some((1_u64 << 53) - 1)); + assert_eq!(sanitized.models.len(), 64); + assert!(sanitized + .models + .iter() + .all(|model| model.model.normalized_name.as_deref() != Some("m0"))); +} + +#[test] +fn sanitized_session_rollup_uses_ingest_allowlist() { + let started = DateTime::parse_from_rfc3339("2026-05-01T00:00:00Z") + .expect("start") + .with_timezone(&Utc); + let mut rollup = sample_session_rollup(started); + rollup.usage.cache_creation_5m_tokens = Some(4); + rollup.usage.cache_creation_1h_tokens = Some(8); + rollup.usage.cache_creation_tokens = Some(12); + rollup.project = Some(ProjectInfo { + project_id: "project_path_only".to_string(), + project_label: Some("Scratch".to_string()), + repo_remote_hash: Some("remote".to_string()), + repo_label: Some("statsai".to_string()), + branch_hash: Some("branch".to_string()), + branch_label: Some("main".to_string()), + path_hash: Some("path".to_string()), + path_label: Some("/Users/example/Scratch".to_string()), + }); + rollup.models = vec![SummaryModelUsage { + model: ModelInfo { + name: Some("GPT-5".to_string()), + normalized_name: Some("gpt-5".to_string()), + provider_model_id: Some("gpt-5".to_string()), + speed: Some("standard".to_string()), + reasoning_level: Some(ReasoningLevel::High), + reasoning_level_raw: Some("high".to_string()), + }, + usage: UsageCounts { + input_tokens: Some(1), + cache_creation_5m_tokens: Some(2), + ..UsageCounts::default() + }, + cost: rollup.cost.clone(), + metrics: Some(SummaryModelMetrics { + generated_tps: Some(SummaryMetricTotals { + samples: 1, + sum: 3.0, + }), + }), + }]; + let json = serde_json::to_value(sanitize_session_rollup_for_sync(rollup)).expect("json"); + assert_allowlist( + &json, + &[ + "schema_version", + "session_id", + "device_id", + "provider", + "source_id", + "provider_account_id", + "started_at", + "ended_at", + "duration_seconds", + "usage", + "requests", + "cost", + "models", + "primary_model", + "total_messages", + "user_messages", + "assistant_messages", + "developer_messages", + "project", + "title", + "title_source", + "updated_at", + ], + ); + assert_allowlist( + &json["usage"], + &[ + "total_tokens", + "input_tokens", + "output_tokens", + "cache_creation_tokens", + "cache_read_tokens", + "reasoning_tokens", + "local_prompt_eval_tokens", + "local_eval_tokens", + "requests", + ], + ); + assert!(json["usage"].get("cache_creation_5m_tokens").is_none()); + assert!(json["usage"].get("cache_creation_1h_tokens").is_none()); + assert_allowlist( + &json["cost"], + &[ + "provider_reported_usd", + "estimated_api_equivalent_usd", + "provider_reported_micro_usd", + "estimated_api_equivalent_micro_usd", + ], + ); + assert!(json["cost"].get("currency").is_none()); + assert!(json["cost"].get("confidence").is_none()); + assert_allowlist( + &json["project"], + &[ + "project_id", + "project_label", + "repo_remote_hash", + "repo_label", + "branch_hash", + "branch_label", + "path_hash", + "path_label", + ], + ); + let model = &json["models"][0]; + assert_allowlist(model, &["model", "usage", "cost", "metrics"]); + assert_allowlist( + &model["model"], + &[ + "normalized_name", + "name", + "provider_model_id", + "speed", + "reasoning_level", + "reasoning_level_raw", + ], + ); + assert_allowlist( + &model["usage"], + &[ + "total_tokens", + "input_tokens", + "output_tokens", + "cache_creation_tokens", + "cache_read_tokens", + "reasoning_tokens", + "local_prompt_eval_tokens", + "local_eval_tokens", + "requests", + ], + ); + assert!(model["usage"].get("cache_creation_5m_tokens").is_none()); + assert_allowlist(&model["metrics"], &["generated_tps"]); + assert_allowlist( + &model["metrics"]["generated_tps"], + &["samples", "avg", "min", "max", "p50", "p95", "sum"], + ); +} + +fn assert_allowlist(value: &serde_json::Value, allowed: &[&str]) { + let object = value.as_object().expect("object"); + for key in object.keys() { + assert!( + allowed.contains(&key.as_str()), + "unexpected key {key} in {value}" + ); + } +} diff --git a/crates/statsai-core/src/types/sync.rs b/crates/statsai-core/src/types/sync.rs index c4d7997..6443812 100644 --- a/crates/statsai-core/src/types/sync.rs +++ b/crates/statsai-core/src/types/sync.rs @@ -6,7 +6,7 @@ use crate::{ SubscriptionId, SummaryId, TaskSpan, TaskVerification, TaskVerificationId, UsageEvent, UsageSummary, WorkItem, WorkItemMember, }; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, Datelike, Duration, Utc}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -222,15 +222,282 @@ pub fn sanitize_summary_for_sync(mut summary: UsageSummary) -> UsageSummary { summary } +/// Ingest caps from `session_rollup.v1`. A value outside these bounds rejects +/// the whole batch, so the snapshot never lands and hosted rows stay put. +const SYNC_TOKEN_MAX: u64 = 1_000_000_000_000; +const SYNC_REQUEST_MAX: u64 = 1_000_000_000; +const SYNC_COST_MINOR_MAX: i64 = 1_000_000_000; +const SYNC_COST_MICRO_MAX: i64 = SYNC_COST_MINOR_MAX * 10_000; +const SYNC_SAFE_INT_MAX: u64 = (1_u64 << 53) - 1; +const SYNC_LABEL_UTF16_MAX: usize = 256; +const SYNC_PATH_LABEL_UTF16_MAX: usize = 1024; +const SYNC_SESSION_MODEL_MAX: usize = 64; +const SYNC_SESSION_PAYLOAD_MAX_BYTES: usize = 64 * 1024; +const SYNC_FUTURE_SKEW: Duration = Duration::seconds(24 * 60 * 60); + /// Drops unstable projects and generic or secret-looking titles before a /// session rollup leaves the device. `source_id` stays, matching summaries. /// Nested usage and cost already serialize to the closed ingest keys. +/// +/// Values the ingest would reject are repaired here. One bad row fails the +/// batch, and session chunks are sent before the snapshot, so a permanent +/// rejection also leaves hosted rows that should have been retired. #[must_use] pub fn sanitize_session_rollup_for_sync(mut rollup: SessionRollupV1) -> SessionRollupV1 { - rollup.project = rollup.project.and_then(sanitize_project_for_sync); - if crate::task_title_is_generic(rollup.title.as_deref()) { - rollup.title = None; - rollup.title_source = None; + clamp_session_timestamps(&mut rollup); + rollup.project = rollup + .project + .and_then(sanitize_project_for_sync) + .map(clamp_project_labels); + clamp_session_title(&mut rollup); + rollup.primary_model = rollup + .primary_model + .and_then(|label| sync_label(&label, SYNC_LABEL_UTF16_MAX)); + clamp_usage_counts(&mut rollup.usage); + rollup.requests = rollup.requests.min(SYNC_REQUEST_MAX); + clamp_cost(&mut rollup.cost); + clamp_message_count(&mut rollup.total_messages); + clamp_message_count(&mut rollup.user_messages); + clamp_message_count(&mut rollup.assistant_messages); + clamp_message_count(&mut rollup.developer_messages); + for model in &mut rollup.models { + clamp_usage_counts(&mut model.usage); + clamp_cost(&mut model.cost); + model.model.name = model + .model + .name + .as_deref() + .and_then(|label| sync_label(label, SYNC_LABEL_UTF16_MAX)); + model.model.normalized_name = model + .model + .normalized_name + .as_deref() + .and_then(|label| sync_label(label, SYNC_LABEL_UTF16_MAX)); + model.model.provider_model_id = model + .model + .provider_model_id + .as_deref() + .and_then(|label| sync_label(label, SYNC_LABEL_UTF16_MAX)); + model.model.speed = model + .model + .speed + .as_deref() + .and_then(|label| sync_label(label, SYNC_LABEL_UTF16_MAX)); + model.model.reasoning_level_raw = model + .model + .reasoning_level_raw + .as_deref() + .and_then(|label| sync_label(label, SYNC_LABEL_UTF16_MAX)); } + if rollup.models.len() > SYNC_SESSION_MODEL_MAX { + sort_models_by_tokens(&mut rollup.models); + rollup.models.truncate(SYNC_SESSION_MODEL_MAX); + } + shrink_session_payload(&mut rollup); rollup } + +fn clamp_session_timestamps(rollup: &mut SessionRollupV1) { + let latest = Utc::now() + SYNC_FUTURE_SKEW; + // Prefer a timestamp already on the row so a clock step does not give the + // repaired payload a new hash on every sync. + let fallback = [rollup.updated_at, rollup.started_at, rollup.ended_at] + .into_iter() + .find(|value| sync_timestamp_ok(*value, latest)) + .unwrap_or_else(|| Utc::now().min(latest)); + rollup.started_at = clamp_sync_timestamp(rollup.started_at, latest, fallback); + rollup.ended_at = clamp_sync_timestamp(rollup.ended_at, latest, fallback); + if rollup.ended_at < rollup.started_at { + rollup.ended_at = rollup.started_at; + } + rollup.updated_at = clamp_sync_timestamp(rollup.updated_at, latest, fallback); + let seconds = rollup + .ended_at + .signed_duration_since(rollup.started_at) + .num_seconds(); + rollup.duration_seconds = u64::try_from(seconds) + .ok() + .map(|value| value.min(SYNC_SAFE_INT_MAX)); +} + +fn sync_timestamp_ok(value: DateTime, latest: DateTime) -> bool { + (1..=9999).contains(&value.year()) && value <= latest +} + +fn clamp_sync_timestamp( + value: DateTime, + latest: DateTime, + fallback: DateTime, +) -> DateTime { + if sync_timestamp_ok(value, latest) { + value + } else { + fallback + } +} + +fn clamp_project_labels(mut project: ProjectInfo) -> ProjectInfo { + project.project_label = project + .project_label + .as_deref() + .and_then(|label| sync_label(label, SYNC_LABEL_UTF16_MAX)); + project.repo_label = project + .repo_label + .as_deref() + .and_then(|label| sync_label(label, SYNC_LABEL_UTF16_MAX)); + project.branch_label = project + .branch_label + .as_deref() + .and_then(|label| sync_label(label, SYNC_LABEL_UTF16_MAX)); + project.path_label = project + .path_label + .as_deref() + .and_then(|label| sync_label(label, SYNC_PATH_LABEL_UTF16_MAX)); + project +} + +fn clamp_session_title(rollup: &mut SessionRollupV1) { + let title = rollup + .title + .as_deref() + .and_then(|label| sync_label(label, SYNC_LABEL_UTF16_MAX)) + .filter(|label| !crate::task_title_is_generic(Some(label))); + if title.is_none() { + rollup.title_source = None; + } + rollup.title = title; +} + +fn clamp_usage_counts(usage: &mut crate::UsageCounts) { + clamp_token(&mut usage.input_tokens); + clamp_token(&mut usage.output_tokens); + clamp_token(&mut usage.cache_creation_tokens); + clamp_token(&mut usage.cache_creation_5m_tokens); + clamp_token(&mut usage.cache_creation_1h_tokens); + clamp_token(&mut usage.cache_read_tokens); + clamp_token(&mut usage.reasoning_tokens); + clamp_token(&mut usage.total_tokens); + clamp_token(&mut usage.local_prompt_eval_tokens); + clamp_token(&mut usage.local_eval_tokens); + if let Some(requests) = usage.requests.as_mut() { + *requests = (*requests).min(SYNC_REQUEST_MAX); + } +} + +fn clamp_token(value: &mut Option) { + if let Some(amount) = value.as_mut() { + *amount = (*amount).min(SYNC_TOKEN_MAX); + } +} + +fn clamp_message_count(value: &mut Option) { + if let Some(amount) = value.as_mut() { + *amount = (*amount).min(SYNC_SAFE_INT_MAX); + } +} + +fn clamp_cost(cost: &mut crate::CostInfo) { + clamp_cost_field(&mut cost.provider_reported_usd, SYNC_COST_MINOR_MAX); + clamp_cost_field(&mut cost.estimated_api_equivalent_usd, SYNC_COST_MINOR_MAX); + clamp_cost_field(&mut cost.provider_reported_micro_usd, SYNC_COST_MICRO_MAX); + clamp_cost_field( + &mut cost.estimated_api_equivalent_micro_usd, + SYNC_COST_MICRO_MAX, + ); +} + +fn clamp_cost_field(value: &mut Option, max: i64) { + if let Some(amount) = value.as_mut() { + *amount = (*amount).clamp(0, max); + } +} + +fn sort_models_by_tokens(models: &mut [crate::SummaryModelUsage]) { + models.sort_by(|left, right| { + right + .usage + .computed_total() + .cmp(&left.usage.computed_total()) + .then_with(|| model_wire_key(&left.model).cmp(&model_wire_key(&right.model))) + }); +} + +fn model_wire_key(model: &crate::ModelInfo) -> String { + model + .normalized_name + .clone() + .or_else(|| model.name.clone()) + .or_else(|| model.provider_model_id.clone()) + .unwrap_or_default() +} + +fn shrink_session_payload(rollup: &mut SessionRollupV1) { + if payload_within_limit(rollup) { + return; + } + if rollup.models.len() > 1 { + sort_models_by_tokens(&mut rollup.models); + } + while rollup.models.len() > 1 && !payload_within_limit(rollup) { + rollup.models.pop(); + } + if payload_within_limit(rollup) { + return; + } + if let Some(project) = rollup.project.as_mut() { + project.path_label = None; + project.project_label = None; + project.repo_label = None; + project.branch_label = None; + } + rollup.title = None; + rollup.title_source = None; + if !payload_within_limit(rollup) { + rollup.models.clear(); + } +} + +fn payload_within_limit(rollup: &SessionRollupV1) -> bool { + serde_json::to_vec(rollup) + .map(|payload| payload.len() <= SYNC_SESSION_PAYLOAD_MAX_BYTES) + .unwrap_or(false) +} + +fn sync_label(value: &str, max_utf16: usize) -> Option { + let cleaned = value + .chars() + .map(|ch| { + if is_disallowed_sync_control(ch) { + ' ' + } else { + ch + } + }) + .collect::(); + let collapsed = cleaned.split_whitespace().collect::>().join(" "); + let truncated = truncate_utf16(collapsed.trim(), max_utf16); + let truncated = truncated.trim(); + if truncated.is_empty() { + None + } else { + Some(truncated.to_string()) + } +} + +fn is_disallowed_sync_control(ch: char) -> bool { + ('\u{0000}'..='\u{001f}').contains(&ch) || ch == '\u{007f}' +} + +fn truncate_utf16(value: &str, max_units: usize) -> String { + let mut units = 0usize; + let mut end = 0usize; + for (index, ch) in value.char_indices() { + let next = units + ch.len_utf16(); + if next > max_units { + break; + } + units = next; + end = index + ch.len_utf8(); + } + value[..end].to_string() +} diff --git a/crates/statsai-store/src/rollups/sessions.rs b/crates/statsai-store/src/rollups/sessions.rs index d22bf1c..5e705e4 100644 --- a/crates/statsai-store/src/rollups/sessions.rs +++ b/crates/statsai-store/src/rollups/sessions.rs @@ -528,11 +528,16 @@ pub(crate) fn build_session_rollup( .map(|event| event.session.started_at) .min() .unwrap_or(oldest.session.started_at); - let ended_at = events + let mut ended_at = events .iter() .map(|event| event.session.ended_at.unwrap_or(event.created_at)) .max() .unwrap_or(newest.created_at); + // A provider clock can record a completion before the session start. + // Ordering the interval keeps duration non-negative. + if ended_at < started_at { + ended_at = started_at; + } let duration_seconds = ended_at .signed_duration_since(started_at) .num_seconds() diff --git a/crates/statsai-store/src/tests/rollups/sessions.rs b/crates/statsai-store/src/tests/rollups/sessions.rs index ebb1ca5..dba8b09 100644 --- a/crates/statsai-store/src/tests/rollups/sessions.rs +++ b/crates/statsai-store/src/tests/rollups/sessions.rs @@ -109,6 +109,45 @@ fn session_rollup_sums_mixed_events() { assert_eq!(rollup.models.len(), 2); } +#[test] +fn session_rollup_orders_inverted_event_times() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-inverted"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 2, 9, 0, 0) + .single() + .expect("start"); + let later = start + chrono::Duration::minutes(12); + let mut event = test_store_event(&source, later, "inverted"); + stamp_session(&mut event, "raw-inverted"); + event.session.started_at = later; + event.session.ended_at = Some(start); + event.created_at = start; + assert!(store.insert_event(&event).expect("insert")); + + let rollups = store + .session_rollups_in_period( + None, + later + chrono::Duration::days(1), + &SessionFilter::default(), + SessionSort::Started, + 10, + 0, + ) + .expect("rollups"); + assert_eq!(rollups.len(), 1); + assert_eq!(rollups[0].started_at, later); + assert_eq!(rollups[0].ended_at, later); + assert_eq!(rollups[0].duration_seconds, Some(0)); +} + #[test] fn session_rollup_refreshes_on_insert_and_delete() { let store = Store::in_memory().expect("store"); From 6ea8c1961653941b198296424739a5eb97cf909d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 26 Sep 2026 08:16:37 +0000 Subject: [PATCH 04/16] Fit session sync batches to the D1 query budget Session projects were left out of the HTTP rollup estimate, and session rows missed the ownership write, dashboard timestamp, and account-alias lookup the API charges. A sessions batch could pass the client budget and come back as HTTP 413. Count those costs, and split session and activity chunks when the server still rejects them. Co-authored-by: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> --- .../statsai/src/cli/sync/chunking/budget.rs | 79 ++++++++++++++- crates/statsai/src/cli/sync/http.rs | 12 ++- .../src/cli/tests/sync/chunking/budget.rs | 93 ++++++++++++++++++ .../src/cli/tests/sync/chunking/payloads.rs | 66 +++++++++++++ crates/statsai/src/cli/tests/sync/http.rs | 44 +++++++++ crates/statsai/src/cli/tests/sync/mod.rs | 96 +++++++++++++++++++ 6 files changed, 384 insertions(+), 6 deletions(-) diff --git a/crates/statsai/src/cli/sync/chunking/budget.rs b/crates/statsai/src/cli/sync/chunking/budget.rs index 65ab38d..821185a 100644 --- a/crates/statsai/src/cli/sync/chunking/budget.rs +++ b/crates/statsai/src/cli/sync/chunking/budget.rs @@ -102,9 +102,9 @@ pub(crate) fn estimate_http_rollup_d1_queries(batch: &SyncBatch) -> usize { // ownership write, matching quota-cycle batching. let activity_queries = usize::from(!batch.activity_rollups.is_empty()) * 2 + usize::from(!batch.activity_coverage.is_empty()) * 2; - // Session rows are one lookup plus one write. A closing snapshot that - // names session_rollup_ids also deletes orphans the device no longer owns. - let session_queries = usize::from(!batch.sessions.is_empty()) * 2 + // Sessions are one freshness lookup, one upsert, and one ownership write. + // A closing snapshot that names session_rollup_ids also deletes orphans. + let session_queries = usize::from(!batch.sessions.is_empty()) * 3 + usize::from( batch .authoritative_snapshot @@ -114,6 +114,20 @@ pub(crate) fn estimate_http_rollup_d1_queries(batch: &SyncBatch) -> usize { && snapshot.session_rollup_ids.is_some() }), ); + // Any session or activity row advances the dashboard revision. Summaries + // already pay for the full snapshot refresh above. + let dashboard_snapshot_timestamp_queries = usize::from( + !batch.sessions.is_empty() + || !batch.activity_rollups.is_empty() + || !batch.activity_coverage.is_empty(), + ); + // Plan and evidence account ids already reserve the account-plan alias + // lookup. Sessions and activity share that one query, so charge it only + // when they name an account and nothing else already did. + let session_activity_alias_queries = usize::from( + !http_rollup_account_plan_names_provider_account(batch) + && http_rollup_session_or_activity_names_provider_account(batch), + ); let code_change_owner_metadata_refresh_queries = usize::from( batch.schema_version == SYNC_BATCH_SCHEMA_VERSION && batch @@ -140,6 +154,8 @@ pub(crate) fn estimate_http_rollup_d1_queries(batch: &SyncBatch) -> usize { + quota_cycle_contribution_queries + activity_queries + session_queries + + dashboard_snapshot_timestamp_queries + + session_activity_alias_queries + code_change_owner_metadata_refresh_queries + estimate_http_rollup_task_queries(batch) + final_sync_bookkeeping_queries @@ -386,12 +402,21 @@ pub(crate) fn http_rollup_project_count(batch: &SyncBatch) -> usize { .summaries .iter() .filter_map(http_rollup_summary_project_key) + .chain( + batch + .sessions + .iter() + .filter_map(|session| session.project.as_ref().and_then(http_rollup_project_key)), + ) .collect::>() .len() } pub(crate) fn http_rollup_summary_project_key(summary: &UsageSummary) -> Option { - let project = summary.project.as_ref()?; + summary.project.as_ref().and_then(http_rollup_project_key) +} + +pub(crate) fn http_rollup_project_key(project: &statsai_core::ProjectInfo) -> Option { if !project_has_stable_identity(project) { return None; } @@ -409,12 +434,26 @@ pub(crate) fn http_rollup_project_location_count(batch: &SyncBatch) -> usize { .summaries .iter() .filter_map(http_rollup_summary_project_location_key) + .chain(batch.sessions.iter().filter_map(|session| { + session + .project + .as_ref() + .and_then(http_rollup_project_location_key) + })) .collect::>() .len() } pub(crate) fn http_rollup_summary_project_location_key(summary: &UsageSummary) -> Option { - let project = summary.project.as_ref()?; + summary + .project + .as_ref() + .and_then(http_rollup_project_location_key) +} + +pub(crate) fn http_rollup_project_location_key( + project: &statsai_core::ProjectInfo, +) -> Option { if !project_has_stable_identity(project) { return None; } @@ -426,3 +465,33 @@ pub(crate) fn http_rollup_summary_project_location_key(summary: &UsageSummary) - } Some(format!("project:{}", project.project_id)) } + +fn http_rollup_account_plan_names_provider_account(batch: &SyncBatch) -> bool { + !unique_non_empty_provider_account_ids( + batch + .account_plan_observations + .iter() + .map(|observation| observation.provider_account_id.0.as_str()) + .chain( + batch + .account_evidence_summaries + .iter() + .map(|summary| summary.provider_account_id.0.as_str()), + ), + ) + .is_empty() +} + +fn http_rollup_session_or_activity_names_provider_account(batch: &SyncBatch) -> bool { + batch.sessions.iter().any(|session| { + session + .provider_account_id + .as_ref() + .is_some_and(|account_id| !account_id.0.trim().is_empty()) + }) || batch.activity_rollups.iter().any(|rollup| { + rollup + .provider_account_id + .as_ref() + .is_some_and(|account_id| !account_id.0.trim().is_empty()) + }) +} diff --git a/crates/statsai/src/cli/sync/http.rs b/crates/statsai/src/cli/sync/http.rs index 5bfb220..a8dbccb 100644 --- a/crates/statsai/src/cli/sync/http.rs +++ b/crates/statsai/src/cli/sync/http.rs @@ -12,7 +12,8 @@ use std::time::Duration as StdDuration; use super::super::args::SyncCommand; use super::batch::{record_rollup_sync_chunk_success, record_sync_batch_success}; use super::chunking::{ - has_non_code_change_payload, has_non_quota_cycle_payload, http_rollup_metadata_count, + has_non_activity_payload, has_non_code_change_payload, has_non_quota_cycle_payload, + has_non_session_payload, http_rollup_metadata_count, split_http_rollup_sync_batch_after_budget_error, split_http_rollup_sync_batches, HttpRollupIndexedChunkKind, HTTP_ROLLUP_SUMMARIES_PER_BATCH, }; @@ -379,6 +380,9 @@ pub(crate) fn should_retry_http_rollup_chunk_after_error( { return false; } + // Relinks rewrite three tables per location. The server decides that after + // reading stored projects, so a 413 on a splittable session or activity + // chunk has to be retried smaller even when the local estimate fit. chunk.summaries.len() > 1 || chunk.sources.len() > 1 || chunk.accounts.len() > 1 @@ -390,10 +394,16 @@ pub(crate) fn should_retry_http_rollup_chunk_after_error( || chunk.task_verifications.len() > 1 || chunk.code_change_metrics.len() > 1 || chunk.quota_cycle_contributions.len() > 1 + || chunk.activity_rollups.len() > 1 + || chunk.activity_coverage.len() > 1 + || chunk.sessions.len() > 1 || (!chunk.task_buckets.is_empty() && !chunk.task_verifications.is_empty()) || (http_rollup_metadata_count(chunk) > 0 && !chunk.summaries.is_empty()) || (!chunk.code_change_metrics.is_empty() && has_non_code_change_payload(chunk)) || (!chunk.quota_cycle_contributions.is_empty() && has_non_quota_cycle_payload(chunk)) + || (!chunk.sessions.is_empty() && has_non_session_payload(chunk)) + || ((!chunk.activity_rollups.is_empty() || !chunk.activity_coverage.is_empty()) + && has_non_activity_payload(chunk)) } fn http_rollup_retry_error_label(error: &anyhow::Error) -> &'static str { diff --git a/crates/statsai/src/cli/tests/sync/chunking/budget.rs b/crates/statsai/src/cli/tests/sync/chunking/budget.rs index 65aa95c..bbc89eb 100644 --- a/crates/statsai/src/cli/tests/sync/chunking/budget.rs +++ b/crates/statsai/src/cli/tests/sync/chunking/budget.rs @@ -556,3 +556,96 @@ fn multi_bucket_dense_task_sync_splits_to_fit_chunked_write_budget() { .iter() .all(|chunk| estimate_http_rollup_d1_queries(chunk) <= HTTP_ROLLUP_D1_QUERY_BUDGET)); } + +#[test] +fn session_project_and_activity_rows_match_the_backend_d1_cost() { + let now = Utc + .with_ymd_and_hms(2026, 6, 20, 12, 0, 0) + .single() + .expect("date"); + let baseline = test_task_only_sync_batch(now, 0, 0); + let baseline_queries = estimate_http_rollup_d1_queries(&baseline); + + let mut bare_session = baseline.clone(); + let mut sessions = test_session_rollups(1); + sessions[0].project = None; + sessions[0].provider_account_id = None; + bare_session.sessions = sessions; + assert_eq!( + estimate_http_rollup_d1_queries(&bare_session), + baseline_queries + 4, + "session lookup, write, ownership, and dashboard timestamp" + ); + + let mut named_session = baseline.clone(); + named_session.sessions = test_session_rollups(1); + assert_eq!(http_rollup_project_count(&named_session), 1); + assert_eq!(http_rollup_project_location_count(&named_session), 1); + assert_eq!( + estimate_http_rollup_d1_queries(&named_session), + baseline_queries + 8, + "project, location, location lookup, and account-alias lookup" + ); + + let mut shared_project = named_session.clone(); + let mut shared_sessions = test_session_rollups(2); + shared_sessions[1].project = shared_sessions[0].project.clone(); + shared_project.sessions = shared_sessions; + assert_eq!(http_rollup_project_count(&shared_project), 1); + assert_eq!(http_rollup_project_location_count(&shared_project), 1); + assert_eq!( + estimate_http_rollup_d1_queries(&shared_project), + estimate_http_rollup_d1_queries(&named_session) + ); + + let mut activity = baseline.clone(); + activity.activity_rollups = vec![test_activity_rollup(0, None)]; + assert_eq!( + estimate_http_rollup_d1_queries(&activity), + baseline_queries + 3, + "activity write, ownership, and dashboard timestamp" + ); + activity.activity_rollups = vec![test_activity_rollup( + 0, + Some(provider_account_id("codex", "personal")), + )]; + assert_eq!( + estimate_http_rollup_d1_queries(&activity), + baseline_queries + 4, + "activity rows with a provider account also pay for the alias lookup" + ); + activity.activity_coverage = vec![test_activity_coverage(0)]; + assert_eq!( + estimate_http_rollup_d1_queries(&activity), + baseline_queries + 6, + "coverage is its own write and ownership; the timestamp and alias stay one query" + ); +} + +#[test] +fn session_batches_with_many_projects_split_under_the_d1_budget() { + let now = Utc + .with_ymd_and_hms(2026, 6, 20, 12, 0, 0) + .single() + .expect("date"); + let mut batch = test_task_only_sync_batch(now, 0, 0); + batch.batch_id = "batch_session_projects".to_string(); + batch.sessions = test_session_rollups(40); + assert_eq!(http_rollup_project_count(&batch), 40); + assert_eq!(http_rollup_project_location_count(&batch), 40); + assert!(estimate_http_rollup_d1_queries(&batch) > HTTP_ROLLUP_D1_QUERY_BUDGET); + + let chunks = split_http_rollup_sync_batches(&batch); + + assert!(chunks.len() > 1); + assert_eq!( + chunks + .iter() + .map(|chunk| chunk.sessions.len()) + .sum::(), + batch.sessions.len() + ); + assert!(chunks + .iter() + .all(|chunk| estimate_http_rollup_d1_queries(chunk) <= HTTP_ROLLUP_D1_QUERY_BUDGET)); +} diff --git a/crates/statsai/src/cli/tests/sync/chunking/payloads.rs b/crates/statsai/src/cli/tests/sync/chunking/payloads.rs index ccfe0ec..80c1765 100644 --- a/crates/statsai/src/cli/tests/sync/chunking/payloads.rs +++ b/crates/statsai/src/cli/tests/sync/chunking/payloads.rs @@ -215,6 +215,72 @@ fn http_rollup_retry_splits_code_change_only_payloads() { ); } +#[test] +fn http_rollup_retry_splits_sessions_and_activity_after_d1_budget_rejection() { + let now = Utc + .with_ymd_and_hms(2026, 6, 20, 12, 0, 0) + .single() + .expect("date"); + let budget_error = anyhow::anyhow!( + r#"sync endpoint returned HTTP 413: {{"error":"sync_batch_d1_query_budget_exceeded","estimatedQueries":70,"maxQueries":45}}"# + ); + let mut sessions = test_task_only_sync_batch(now, 0, 0); + sessions.sessions = test_session_rollups(2); + assert!(should_retry_http_rollup_chunk_after_error( + &sessions, + &budget_error + )); + let session_chunks = split_http_rollup_sync_batch_after_budget_error(&sessions); + assert_eq!(session_chunks.len(), 2); + assert!(session_chunks.iter().all(|chunk| chunk.sessions.len() == 1)); + + let mut one_session = sessions.clone(); + one_session.sessions.truncate(1); + assert!(!should_retry_http_rollup_chunk_after_error( + &one_session, + &budget_error + )); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/codex-http-session-mixed"), + LocationOrigin::Configured, + ); + one_session.summaries = vec![test_summary("codex", &source, now, 10, None)]; + assert!(should_retry_http_rollup_chunk_after_error( + &one_session, + &budget_error + )); + + let mut activity = test_task_only_sync_batch(now, 0, 0); + activity.activity_rollups = vec![test_activity_rollup(0, None), test_activity_rollup(1, None)]; + assert!(should_retry_http_rollup_chunk_after_error( + &activity, + &budget_error + )); + let activity_chunks = split_http_rollup_sync_batch_after_budget_error(&activity); + assert_eq!(activity_chunks.len(), 2); + assert!(activity_chunks + .iter() + .all(|chunk| chunk.activity_rollups.len() == 1)); + + let mut coverage = test_task_only_sync_batch(now, 0, 0); + coverage.activity_coverage = vec![test_activity_coverage(0), test_activity_coverage(1)]; + assert!(should_retry_http_rollup_chunk_after_error( + &coverage, + &budget_error + )); + + let mut mixed_activity = test_task_only_sync_batch(now, 0, 0); + mixed_activity.activity_rollups = vec![test_activity_rollup(0, None)]; + mixed_activity.summaries = vec![test_summary("codex", &source, now, 10, None)]; + assert!(should_retry_http_rollup_chunk_after_error( + &mixed_activity, + &budget_error + )); +} + #[test] fn http_rollup_retry_halves_task_only_bucket_chunks() { let now = Utc diff --git a/crates/statsai/src/cli/tests/sync/http.rs b/crates/statsai/src/cli/tests/sync/http.rs index e51c4d3..9a4dc6a 100644 --- a/crates/statsai/src/cli/tests/sync/http.rs +++ b/crates/statsai/src/cli/tests/sync/http.rs @@ -112,6 +112,50 @@ fn http_rollup_sync_retries_smaller_batches_after_budget_rejection() { assert!(pending.is_empty()); } +#[test] +fn http_rollup_sync_retries_sessions_after_d1_budget_rejection() { + let store = Store::in_memory().expect("store"); + let endpoint = "https://api.example.com/api/sync/batches".to_string(); + let now = Utc + .with_ymd_and_hms(2026, 6, 20, 12, 0, 0) + .single() + .expect("date"); + let mut batch = test_task_only_sync_batch(now, 0, 0); + batch.batch_id = "batch_sessions".to_string(); + batch.sessions = test_session_rollups(4); + let logical_batch_id = logical_http_rollup_batch_id(&batch.batch_id).to_string(); + let observed = Arc::new(Mutex::new(Vec::new())); + let observed_for_send = Arc::clone(&observed); + + send_http_rollup_chunk_with_retry_using(&batch, &|chunk| { + observed_for_send + .lock() + .expect("observed lock") + .push((chunk.batch_id.clone(), chunk.sessions.len())); + if chunk.sessions.len() > 2 { + return Err(anyhow::Error::msg( + r#"sync endpoint returned HTTP 413: {"error":"sync_batch_d1_query_budget_exceeded","estimatedQueries":70,"maxQueries":45}"#, + )); + } + record_rollup_sync_chunk_success(&store, "http", &endpoint, &logical_batch_id, chunk) + }) + .expect("send"); + + let observed = observed.lock().expect("observed lock").clone(); + assert_eq!( + observed, + vec![ + ("batch_sessions".to_string(), 4), + ("batch_sessions_sessions_1".to_string(), 2), + ("batch_sessions_sessions_2".to_string(), 2), + ] + ); + let pending = store + .pending_session_rollups_for_sync("http", &endpoint, &batch.sessions) + .expect("pending sessions"); + assert!(pending.is_empty()); +} + #[test] fn http_rollup_sync_retries_smaller_batches_after_payload_too_large() { let store = Store::in_memory().expect("store"); diff --git a/crates/statsai/src/cli/tests/sync/mod.rs b/crates/statsai/src/cli/tests/sync/mod.rs index 174ef7d..9ac0fcd 100644 --- a/crates/statsai/src/cli/tests/sync/mod.rs +++ b/crates/statsai/src/cli/tests/sync/mod.rs @@ -320,6 +320,102 @@ pub(crate) fn test_task_only_sync_batch( } } +pub(crate) fn test_session_rollups(count: usize) -> Vec { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/codex-http-session-budget"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let now = Utc + .with_ymd_and_hms(2026, 6, 20, 12, 0, 0) + .single() + .expect("date"); + let account_id = provider_account_id("codex", "personal"); + for index in 0..count { + let mut event = test_event( + "codex", + &source, + now + Duration::minutes(index as i64), + Some(account_id.clone()), + TokenParts::total(10), + ); + event.session.session_id = format!("session-budget-{index}"); + event.project = Some(ProjectInfo { + project_id: format!("project-budget-{index}"), + project_label: Some(format!("Project {index}")), + repo_remote_hash: Some(format!("repo-hash-{index}")), + repo_label: Some(format!("owner/repo-{index}")), + branch_hash: None, + branch_label: None, + path_hash: Some(format!("path-hash-{index}")), + path_label: Some(format!("/tmp/project-{index}")), + }); + store.insert_event(&event).expect("event"); + } + let rollups = store.all_session_rollups().expect("session rollups"); + assert_eq!(rollups.len(), count); + rollups +} + +pub(crate) fn test_activity_rollup( + index: usize, + provider_account_id: Option, +) -> statsai_core::ActivityRollupV1 { + let seen = Utc + .with_ymd_and_hms(2026, 6, 20, 12, 0, 0) + .single() + .expect("date"); + statsai_core::ActivityRollupV1 { + schema_version: statsai_core::ACTIVITY_ROLLUP_SCHEMA_VERSION.to_string(), + rollup_id: format!("activity-rollup-{index}"), + device_id: "device".to_string(), + source_id: SourceId("source".to_string()), + provider: "codex".to_string(), + provider_account_id, + day: "2026-06-20".to_string(), + kind: statsai_core::ActivityKind::Tool, + entity_key: "shell".to_string(), + display_name: "shell".to_string(), + family: statsai_core::ActivityFamily::Shell, + mcp_server: None, + mcp_tool: None, + plugin: None, + skill_catalog: None, + model: None, + calls: 1, + succeeded: 1, + failed: 0, + unknown: 0, + duration_samples: 1, + duration_sum_ms: 10, + duration_max_ms: Some(10), + duration_kind: Some(statsai_core::ActivityDurationKind::Reported), + first_seen: seen, + last_seen: seen, + evidence: "codex-native-items".to_string(), + } +} + +pub(crate) fn test_activity_coverage(index: usize) -> statsai_core::ActivityCoverageV1 { + statsai_core::ActivityCoverageV1 { + schema_version: statsai_core::ACTIVITY_COVERAGE_SCHEMA_VERSION.to_string(), + coverage_id: format!("activity-coverage-{index}"), + device_id: "device".to_string(), + source_id: SourceId("source".to_string()), + provider: "codex".to_string(), + day: "2026-06-20".to_string(), + day_end: "2026-06-20".to_string(), + kind: statsai_core::ActivityKind::Tool, + level: statsai_core::ActivityCoverageLevel::Complete, + evidence: "codex-native-items".to_string(), + parser_revision: statsai_core::ACTIVITY_PARSER_REVISION.to_string(), + } +} + pub(crate) fn test_code_change_metric( index: usize, now: DateTime, From 0ecb0f23b372a819f4c22c5d9e73a039de767426 Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Sat, 26 Sep 2026 12:39:45 +0300 Subject: [PATCH 05/16] Keep session rollups to real sessions with provider titles Cursor's usage export names no session, so every local row carried its own record hash as a session id and listed one request as one session. Events keyed by their own provider record no longer form sessions. A session with a single timestamp and no recorded end now has an unknown duration instead of 0s, and unknown durations sync as null rather than pulling the median to zero. Titles now come from the providers' own session names: Codex thread names load whether or not tasks are collected, Claude Code custom-title and ai-title records name every event in their session, and older Codex task spans keyed by rollout file stem join on the UUID the stem ends in. The session backfill key moves to v2 so existing stores rebuild once. The rebuild keeps rows whose content is unchanged, so only sessions it changed are resent. --- README.md | 7 +- crates/statsai-adapters/src/claude/parse.rs | 49 ++++- .../src/claude/tests/parse.rs | 44 +++++ crates/statsai-adapters/src/codex/mod.rs | 7 +- .../statsai-adapters/src/codex/parse/mod.rs | 15 +- .../src/codex/tests/parse/session.rs | 10 + crates/statsai-core/src/tests/types.rs | 15 +- crates/statsai-core/src/types/session.rs | 23 +-- crates/statsai-core/src/types/sync.rs | 5 + crates/statsai-store/src/rollups/sessions.rs | 111 +++++++++-- .../src/tests/rollups/sessions.rs | 174 ++++++++++++++++++ .../tests/cli_surface/schema-sync-batch.json | 8 +- docs/sync-contract.md | 5 +- 13 files changed, 412 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index bdc5a5e..ebe6128 100644 --- a/README.md +++ b/README.md @@ -166,8 +166,11 @@ include bounded conversation-derived task titles, summary previews, and todo excerpts. Hosted session sync is separate and off by default. `statsai sync --include-sessions` adds `sessions` rows to `sync_batch.v6`: hashed session ids, token and request totals, duration, message counts, and a -bounded title. `--exclude-sessions` stops sending them and leaves sessions -already hosted in place. You can inspect the exact sync contract in +bounded title. Titles come from the provider's own session name (Codex thread +names, Claude Code and OpenCode session titles) or a local task title. Cursor's +usage export names no session, so its local rows are not sessions. +`--exclude-sessions` stops sending them and leaves sessions already hosted in +place. You can inspect the exact sync contract in [`docs/sync-contract.md`](docs/sync-contract.md) and verify the resolved sync target with: diff --git a/crates/statsai-adapters/src/claude/parse.rs b/crates/statsai-adapters/src/claude/parse.rs index 53a88c5..9f49c7b 100644 --- a/crates/statsai-adapters/src/claude/parse.rs +++ b/crates/statsai-adapters/src/claude/parse.rs @@ -17,9 +17,9 @@ use anyhow::{Context, Result}; use chrono::Utc; use serde_json::Value; use statsai_core::{ - canonical_display, expand_home_path, hash_text, summary_id, Confidence, EventSource, - IdentitySource, ParseEvidence, ProjectInfo, SourceKind, SourceLocation, SummaryMetadata, - UsageCounts, UsageSummary, USAGE_SUMMARY_SCHEMA_VERSION, + canonical_display, expand_home_path, hash_text, summarize_task_text, summary_id, Confidence, + EventSource, IdentitySource, ParseEvidence, ProjectInfo, SourceKind, SourceLocation, + SummaryMetadata, UsageCounts, UsageSummary, USAGE_SUMMARY_SCHEMA_VERSION, }; use statsai_pricing::{estimate_cost_at, pricing_changes_between, unknown_cost}; use std::collections::HashMap; @@ -48,6 +48,8 @@ pub(crate) fn parse_claude_file( // Later snapshots of one provider record replace the event. User lines // counted on the first snapshot have to ride along with that replacement. let mut last_usage_users = HashMap::::new(); + let mut session_titles = HashMap::::new(); + let events_start = ctx.scan.events.len(); loop { let line_status = read_bounded_jsonl_line(&mut reader, &mut line_bytes, MAX_JSONL_RECORD_BYTES)?; @@ -82,6 +84,15 @@ pub(crate) fn parse_claude_file( current_reasoning = reasoning; } let session_raw = claude_session_raw(&value, path); + if let Some((rank, title)) = claude_session_title_from_value(&value) { + let replaces = session_titles + .get(&session_raw) + .is_none_or(|(current, _)| rank >= *current); + if replaces { + session_titles.insert(session_raw, (rank, title)); + } + continue; + } let Some(usage_value) = value .pointer("/message/usage") .or_else(|| value.get("usage")) @@ -179,6 +190,25 @@ pub(crate) fn parse_claude_file( push_deduped(ctx.scan, ctx.seen, event, duplicate_selection); } + // Title records can follow the usage they name, so titles land once the + // whole file has been read. + if !session_titles.is_empty() { + let titles_by_hash = session_titles + .into_iter() + .map(|(session_raw, (_, title))| (hash_text(&session_raw), title)) + .collect::>(); + for event in &mut ctx.scan.events[events_start..] { + if let Some(title) = event + .session + .local_session_id_hash + .as_deref() + .and_then(|hash| titles_by_hash.get(hash)) + { + event.session.title = Some(title.clone()); + } + } + } + let activity_started_at = std::time::Instant::now(); activity.finish( ctx.scan, @@ -191,6 +221,19 @@ pub(crate) fn parse_claude_file( Ok(()) } +/// Claude Code names a session with `custom-title` records, which a rename +/// also writes, and occasionally `ai-title`. The later record of the stronger +/// kind wins; the rank says which kind that is. +fn claude_session_title_from_value(value: &Value) -> Option<(u8, String)> { + let (rank, key) = match value.get("type").and_then(Value::as_str)? { + "custom-title" => (1, "customTitle"), + "ai-title" => (0, "aiTitle"), + _ => return None, + }; + let title = summarize_task_text(value.get(key).and_then(Value::as_str), 90)?; + Some((rank, title)) +} + fn claude_session_raw(value: &Value, path: &Path) -> String { value .get("sessionId") diff --git a/crates/statsai-adapters/src/claude/tests/parse.rs b/crates/statsai-adapters/src/claude/tests/parse.rs index 7f11232..3d0bfd2 100644 --- a/crates/statsai-adapters/src/claude/tests/parse.rs +++ b/crates/statsai-adapters/src/claude/tests/parse.rs @@ -1171,3 +1171,47 @@ fn claude_scan_separates_oversized_rows_from_malformed_ones() { ); assert_eq!(scan.events.len(), 1, "the usable row is still collected"); } + +#[test] +fn claude_session_titles_name_every_event_in_the_session() { + let dir = tempfile::tempdir().expect("tempdir"); + let projects = dir.path().join("projects"); + std::fs::create_dir_all(&projects).expect("projects"); + let mut file = File::create(projects.join("session.jsonl")).expect("session file"); + for line in [ + r#"{"type":"ai-title","aiTitle":"Generated name","sessionId":"titled"}"#, + r#"{"timestamp":"2026-05-01T00:00:00Z","sessionId":"titled","message":{"id":"m1","usage":{"input_tokens":1,"output_tokens":2}}}"#, + r#"{"timestamp":"2026-05-01T00:01:00Z","sessionId":"untitled","message":{"id":"m2","usage":{"input_tokens":1,"output_tokens":2}}}"#, + // A custom title outranks the generated one even though it comes later. + r#"{"type":"custom-title","customTitle":"Repair the session indexer","sessionId":"titled"}"#, + r#"{"type":"ai-title","aiTitle":"Later generated name","sessionId":"titled"}"#, + ] { + writeln!(file, "{line}").expect("write line"); + } + + let source = SourceLocation::local_adapter( + CLAUDE_CODE_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let scan = scan_claude_source(&ClaudeCodeAdapter, &source, &options()).expect("scan"); + + assert_eq!(scan.events.len(), 2); + let title_for = |session_raw: &str| { + let hash = statsai_core::hash_text(session_raw); + scan.events + .iter() + .find(|event| event.session.local_session_id_hash.as_deref() == Some(hash.as_str())) + .expect("event") + .session + .title + .clone() + }; + assert_eq!( + title_for("titled").as_deref(), + Some("Repair the session indexer") + ); + assert_eq!(title_for("untitled"), None); +} diff --git a/crates/statsai-adapters/src/codex/mod.rs b/crates/statsai-adapters/src/codex/mod.rs index 32214ea..e6a2655 100644 --- a/crates/statsai-adapters/src/codex/mod.rs +++ b/crates/statsai-adapters/src/codex/mod.rs @@ -123,11 +123,8 @@ pub(crate) fn scan_codex_source( let source_path = PathBuf::from(path_label); let root = codex_source_root(&source_path); let cache_namespaces = scan_cache_namespaces(source, adapter.version()); - let thread_titles = if options.should_collect_tasks() { - load_codex_thread_titles(&root) - } else { - HashMap::new() - }; + // Thread names title sessions as well as tasks, so they load either way. + let thread_titles = load_codex_thread_titles(&root); let mut indexed_candidates = Vec::new(); for (index, candidate) in codex_jsonl_candidates(source, &source_path, &cache_namespaces)? .into_iter() diff --git a/crates/statsai-adapters/src/codex/parse/mod.rs b/crates/statsai-adapters/src/codex/parse/mod.rs index 3b2edc2..ad109c1 100644 --- a/crates/statsai-adapters/src/codex/parse/mod.rs +++ b/crates/statsai-adapters/src/codex/parse/mod.rs @@ -341,10 +341,10 @@ pub(crate) fn parse_codex_file( session_raw = declared_session_id; } if collect_tasks { - current_thread_id = declared_session_id; - let session_id = current_thread_id - .clone() - .or_else(|| Some(session_raw.clone())); + current_thread_id = declared_session_id.clone(); + } + { + let session_id = declared_session_id.or_else(|| Some(session_raw.clone())); current_title = value .pointer("/payload/thread_name") .and_then(Value::as_str) @@ -694,7 +694,7 @@ pub(crate) fn parse_codex_file( .map(|_| LatencySource::Explicit) .or_else(|| duration_ms.map(|_| LatencySource::Inferred)); let time_to_first_token_ms = record.time_to_first_token_ms; - let event = usage_event( + let mut event = usage_event( ctx.adapter, ctx.source, ctx.options, @@ -734,6 +734,7 @@ pub(crate) fn parse_codex_file( dedupe_salt: None, }, ); + event.session.title = turn.title.clone(); let mut linked_quota_lines = turn.usage_lines.clone(); linked_quota_lines.extend_from_slice(&turn.quota_lines); if record.usage.is_some() { @@ -930,7 +931,8 @@ pub(crate) fn parse_codex_file( if consumed_usage_lines.contains(&record.line_number) { continue; } - let event = usage_event( + let session_title = record.session_title; + let mut event = usage_event( ctx.adapter, ctx.source, ctx.options, @@ -966,6 +968,7 @@ pub(crate) fn parse_codex_file( dedupe_salt: None, }, ); + event.session.title = session_title; // A record's paired token_count carries the quota sample for it. let mut linked_quota_lines = vec![record.line_number]; linked_quota_lines.extend(paired_quota_lines.get(&record.line_number)); diff --git a/crates/statsai-adapters/src/codex/tests/parse/session.rs b/crates/statsai-adapters/src/codex/tests/parse/session.rs index 61e7073..82b5d29 100644 --- a/crates/statsai-adapters/src/codex/tests/parse/session.rs +++ b/crates/statsai-adapters/src/codex/tests/parse/session.rs @@ -394,4 +394,14 @@ fn codex_task_spans_capture_thread_id_from_session_meta() { // account bindings all meet on one key instead of a file path. assert_eq!(scan.task_spans[0].session_id.as_deref(), Some("thread-123")); assert_eq!(scan.task_spans[0].title, "Fix parser bug"); + + // The thread name titles the session even when tasks are not collected. + let without_tasks = + scan_codex_source(&CodexAdapter, &source, &options_without_tasks()).expect("scan"); + assert!(without_tasks.task_spans.is_empty()); + assert!(!without_tasks.events.is_empty()); + assert!(without_tasks + .events + .iter() + .all(|event| event.session.title.as_deref() == Some("Fix parser bug"))); } diff --git a/crates/statsai-core/src/tests/types.rs b/crates/statsai-core/src/tests/types.rs index d41b1e3..72cbd80 100644 --- a/crates/statsai-core/src/tests/types.rs +++ b/crates/statsai-core/src/tests/types.rs @@ -144,7 +144,7 @@ fn session_rollup_uses_snake_case_contract() { } #[test] -fn session_rollup_unknown_duration_serializes_as_zero() { +fn session_rollup_unknown_duration_serializes_as_null() { let started = DateTime::parse_from_rfc3339("2026-05-01T00:00:00Z") .expect("start") .with_timezone(&Utc); @@ -182,10 +182,12 @@ fn session_rollup_unknown_duration_serializes_as_zero() { updated_at: started, }; let json = serde_json::to_value(&rollup).expect("serialize"); - assert_eq!(json["duration_seconds"], 0); - rollup.duration_seconds = Some(0); + assert!(json["duration_seconds"].is_null()); let restored: SessionRollupV1 = serde_json::from_value(json).expect("deserialize"); assert_eq!(restored, rollup); + rollup.duration_seconds = Some(0); + let json = serde_json::to_value(&rollup).expect("serialize measured"); + assert_eq!(json["duration_seconds"], 0); } #[test] @@ -652,11 +654,16 @@ fn sanitize_session_rollup_repairs_values_ingest_rejects() { .with_timezone(&Utc); let mut inverted = sample_session_rollup(started + chrono::Duration::hours(2)); inverted.ended_at = started; - inverted.duration_seconds = None; + inverted.duration_seconds = Some(7_200); let sanitized = sanitize_session_rollup_for_sync(inverted); assert_eq!(sanitized.ended_at, sanitized.started_at); assert_eq!(sanitized.duration_seconds, Some(0)); + let mut unknown = sample_session_rollup(started); + unknown.duration_seconds = None; + let sanitized = sanitize_session_rollup_for_sync(unknown); + assert_eq!(sanitized.duration_seconds, None); + let mut future = sample_session_rollup(started); future.started_at = Utc::now() + chrono::Duration::hours(48); future.ended_at = future.started_at; diff --git a/crates/statsai-core/src/types/session.rs b/crates/statsai-core/src/types/session.rs index c34c157..2b5a0a5 100644 --- a/crates/statsai-core/src/types/session.rs +++ b/crates/statsai-core/src/types/session.rs @@ -31,13 +31,8 @@ pub struct SessionRollupV1 { pub provider_account_id: Option, pub started_at: DateTime, pub ended_at: DateTime, - /// Always a non-negative integer on the wire. Unknown durations serialize as 0 - /// because `session_rollup.v1` rejects null. - #[serde( - serialize_with = "serialize_duration_seconds", - deserialize_with = "deserialize_duration_seconds" - )] - #[schemars(with = "u64")] + /// `None` when the session recorded only a start. Unknown durations go out + /// as null; ingest stores them without an end so they stay out of medians. pub duration_seconds: Option, /// Closed usage object. Cache-lifetime splits stay local; the ingest allowlist /// does not include them. @@ -192,20 +187,6 @@ impl From for SummaryModelUsage { } } -fn serialize_duration_seconds(value: &Option, serializer: S) -> Result -where - S: serde::Serializer, -{ - serializer.serialize_u64(value.unwrap_or(0)) -} - -fn deserialize_duration_seconds<'de, D>(deserializer: D) -> Result, D::Error> -where - D: serde::Deserializer<'de>, -{ - Option::::deserialize(deserializer) -} - mod session_usage_serde { use super::{SessionUsageWire, UsageCounts}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; diff --git a/crates/statsai-core/src/types/sync.rs b/crates/statsai-core/src/types/sync.rs index 6443812..5b12873 100644 --- a/crates/statsai-core/src/types/sync.rs +++ b/crates/statsai-core/src/types/sync.rs @@ -311,6 +311,11 @@ fn clamp_session_timestamps(rollup: &mut SessionRollupV1) { rollup.ended_at = rollup.started_at; } rollup.updated_at = clamp_sync_timestamp(rollup.updated_at, latest, fallback); + // An unknown duration stays unknown; clamped timestamps must not turn it + // into a measured zero. + if rollup.duration_seconds.is_none() { + return; + } let seconds = rollup .ended_at .signed_duration_since(rollup.started_at) diff --git a/crates/statsai-store/src/rollups/sessions.rs b/crates/statsai-store/src/rollups/sessions.rs index 5e705e4..5d02fbb 100644 --- a/crates/statsai-store/src/rollups/sessions.rs +++ b/crates/statsai-store/src/rollups/sessions.rs @@ -12,7 +12,11 @@ use statsai_core::{ }; use std::collections::{BTreeMap, BTreeSet}; -const SESSION_ROLLUPS_BACKFILL_KEY: &str = "session_rollups_backfilled_v1"; +// v2 drops record-keyed rows, leaves single-instant durations unknown, and +// joins Codex rollout-stem span titles. +const SESSION_ROLLUPS_BACKFILL_KEY: &str = "session_rollups_backfilled_v2"; +/// Matches the adapters' `provider_record_usage_event.v1` event key version. +const PROVIDER_RECORD_ID_PREFIX: &str = "provider_record_usage_event.v1:"; const UNASSIGNED_ACCOUNT: &str = "unassigned"; #[derive(Debug, Clone, Default, PartialEq, Eq)] @@ -63,13 +67,26 @@ impl Store { let events = self.events()?; let mut grouped: BTreeMap> = BTreeMap::new(); for event in events { + if event_has_own_record_session(&event) { + continue; + } grouped .entry(event.session.session_id.clone()) .or_default() .push(event); } self.with_immediate_transaction(|| { - self.conn.execute("DELETE FROM session_rollups", [])?; + // Rows whose content is unchanged keep their sync state, so a + // rebuild after a builder change resends only what it changed. + let existing = self.session_rollup_payload_hashes()?; + for session_id in existing.keys() { + if !grouped.contains_key(session_id) { + self.conn.execute( + "DELETE FROM session_rollups WHERE session_id = ?1", + params![session_id], + )?; + } + } let hashes = grouped .values() .flatten() @@ -77,9 +94,12 @@ impl Store { .collect::>(); let titles = self.session_titles_for_hashes(&hashes)?; let updated_at = Utc::now(); - for events in grouped.values() { + for (session_id, events) in &grouped { let rollup = build_session_rollup(events, &titles, updated_at); let payload_hash = session_rollup_content_hash(&rollup)?; + if existing.get(session_id) == Some(&payload_hash) { + continue; + } self.write_session_rollup(&rollup, &payload_hash, 1)?; } Ok(grouped.len() as u64) @@ -138,6 +158,7 @@ impl Store { .iter() .map(|raw_id| raw_id.trim()) .filter(|raw_id| !raw_id.is_empty()) + .flat_map(|raw_id| std::iter::once(raw_id).chain(codex_rollout_session_uuid(raw_id))) .map(hash_text) .collect::>(); let session_ids = self.session_ids_for_local_hashes(&hashes)?; @@ -336,11 +357,24 @@ impl Store { let rows = statement.query_map(params![session_id], |row| row.get::<_, String>(0))?; let mut events = Vec::new(); for row in rows { - events.push(serde_json::from_str(&row?)?); + let event: UsageEvent = serde_json::from_str(&row?)?; + if !event_has_own_record_session(&event) { + events.push(event); + } } Ok(events) } + fn session_rollup_payload_hashes(&self) -> Result> { + let mut statement = self + .conn + .prepare("SELECT session_id, payload_hash FROM session_rollups")?; + let rows = statement.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })?; + Ok(rows.collect::, _>>()?) + } + fn session_ids_for_local_hashes(&self, hashes: &[String]) -> Result> { let mut session_ids = BTreeSet::new(); if hashes.is_empty() { @@ -418,13 +452,17 @@ impl Store { let Some(session_id) = session_id.filter(|value| !value.trim().is_empty()) else { continue; }; - remember_hashed_title( - &mut titles, - &wanted, - &session_id, - title.as_deref(), - SessionTitleSource::TaskSpan, - ); + let raw_ids = + std::iter::once(session_id.as_str()).chain(codex_rollout_session_uuid(&session_id)); + for raw_id in raw_ids { + remember_hashed_title( + &mut titles, + &wanted, + raw_id, + title.as_deref(), + SessionTitleSource::TaskSpan, + ); + } } Ok(titles) } @@ -538,11 +576,21 @@ pub(crate) fn build_session_rollup( if ended_at < started_at { ended_at = started_at; } - let duration_seconds = ended_at - .signed_duration_since(started_at) - .num_seconds() - .try_into() - .ok(); + // One instant with no recorded end is a start, not a zero-length session. + // Counting it as 0s pulled the median to zero for every source that + // stamps a single time per row. + let has_recorded_end = events + .iter() + .any(|event| event.session.ended_at.is_some() || event.session.duration_seconds.is_some()); + let duration_seconds = if has_recorded_end || ended_at > started_at { + ended_at + .signed_duration_since(started_at) + .num_seconds() + .try_into() + .ok() + } else { + None + }; let provider_account_id = events .iter() .rev() @@ -619,6 +667,37 @@ fn resolve_title( (None, None) } +/// An event whose session identity is its own provider record belongs to no +/// session. Cursor's usage export names no session, so each local row is keyed +/// by itself, and treating those rows as sessions listed every request as one. +pub(crate) fn event_has_own_record_session(event: &UsageEvent) -> bool { + let Some(session_hash) = event.session.local_session_id_hash.as_deref() else { + return false; + }; + event + .parse_evidence + .as_ref() + .and_then(|evidence| evidence.source_record_id.as_deref()) + .and_then(|record_id| record_id.strip_prefix(PROVIDER_RECORD_ID_PREFIX)) + .and_then(|rest| rest.rsplit_once(':')) + .is_some_and(|(_, record_hash)| record_hash == session_hash) +} + +/// Older Codex task spans name their session by the rollout file stem, while +/// events hash the UUID the file declares. The stem ends in that UUID. +fn codex_rollout_session_uuid(raw_id: &str) -> Option<&str> { + let stem = raw_id.rsplit('/').next()?; + if !stem.starts_with("rollout-") { + return None; + } + let uuid = stem.get(stem.len().checked_sub(36)?..)?; + let is_uuid = uuid.char_indices().all(|(index, character)| match index { + 8 | 13 | 18 | 23 => character == '-', + _ => character.is_ascii_hexdigit(), + }); + is_uuid.then_some(uuid) +} + fn acceptable_session_title(value: Option<&str>) -> Option { let trimmed = value?.trim(); if trimmed.is_empty() || task_title_is_generic(Some(trimmed)) { diff --git a/crates/statsai-store/src/tests/rollups/sessions.rs b/crates/statsai-store/src/tests/rollups/sessions.rs index dba8b09..6429d60 100644 --- a/crates/statsai-store/src/tests/rollups/sessions.rs +++ b/crates/statsai-store/src/tests/rollups/sessions.rs @@ -364,3 +364,177 @@ fn task_span( duration_seconds: Some(0), } } + +fn record_evidence(record_id: &str) -> ParseEvidence { + ParseEvidence { + event_key_version: "provider_record_usage_event.v1".to_string(), + source_file_path_hash: None, + source_line_number: None, + source_record_id: Some(record_id.to_string()), + model_inferred: false, + timestamp_inferred: false, + account_identity_source: IdentitySource::Unresolved, + } +} + +#[test] +fn record_keyed_events_do_not_form_sessions() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "cursor", + "test", + "0", + Path::new("/tmp/session-cursor"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 5, 9, 0, 0) + .single() + .expect("start"); + + // A local Cursor row is keyed by its own record, so its session hash is + // the record hash. + let mut row = test_store_event(&source, start, "cursor-row"); + stamp_session(&mut row, "cursor-row-identity"); + let row_hash = hash_text("cursor-row-identity"); + row.parse_evidence = Some(record_evidence(&format!( + "provider_record_usage_event.v1:cursor_usage_event:{row_hash}" + ))); + store.insert_event(&row).expect("row"); + + // A cloud agent groups its rows under one agent id. + let mut agent = test_store_event(&source, start, "cursor-agent-row"); + stamp_session(&mut agent, "cursor_agent:agent-1"); + let record_hash = hash_text("agent-row-identity"); + agent.parse_evidence = Some(record_evidence(&format!( + "provider_record_usage_event.v1:cursor_usage_event:{record_hash}" + ))); + store.insert_event(&agent).expect("agent"); + + let sessions = store.dirty_session_rollups().expect("sessions"); + assert_eq!(sessions.len(), 1); + assert_eq!(sessions[0].session_id, agent.session.session_id); + + assert_eq!(store.rebuild_session_rollups().expect("rebuild"), 1); + let rebuilt = store.dirty_session_rollups().expect("rebuilt"); + assert_eq!(rebuilt.len(), 1); + assert_eq!(rebuilt[0].session_id, agent.session.session_id); +} + +#[test] +fn single_instant_sessions_have_unknown_duration() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-instant"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 6, 9, 0, 0) + .single() + .expect("start"); + let mut lone = test_store_event(&source, start, "lone"); + stamp_session(&mut lone, "lone-session"); + lone.created_at = start; + store.insert_event(&lone).expect("lone"); + + let mut spanned = test_store_event(&source, start, "spanned-first"); + stamp_session(&mut spanned, "spanned-session"); + spanned.created_at = start; + store.insert_event(&spanned).expect("first"); + let mut later = test_store_event( + &source, + start + chrono::Duration::minutes(3), + "spanned-last", + ); + stamp_session(&mut later, "spanned-session"); + later.created_at = start + chrono::Duration::minutes(3); + store.insert_event(&later).expect("last"); + + let sessions = store.dirty_session_rollups().expect("sessions"); + let duration = |session_id: &str| { + sessions + .iter() + .find(|session| session.session_id == session_id) + .expect("session") + .duration_seconds + }; + assert_eq!(duration(&lone.session.session_id), None); + assert_eq!(duration(&spanned.session.session_id), Some(180)); + let stats = store + .session_stats_in_period( + None, + start + chrono::Duration::days(1), + &SessionFilter::default(), + ) + .expect("stats"); + assert_eq!(stats.median_duration_seconds, Some(180)); +} + +#[test] +fn codex_rollout_stem_span_titles_join_uuid_sessions() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-codex-stem"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 7, 9, 0, 0) + .single() + .expect("start"); + let uuid = "019c537d-7d4b-7c33-bfc9-785a18e58f8c"; + let mut event = test_store_event(&source, start, "codex-stem"); + stamp_session(&mut event, uuid); + store.insert_event(&event).expect("insert"); + + let title = "Repair the rollout title join"; + store + .upsert_task_spans(&[task_span( + &source, + &format!("2026/02/12/rollout-2026-02-12T23-14-18-{uuid}"), + title, + start, + )]) + .expect("span"); + store.rebuild_session_rollups().expect("rebuild"); + let sessions = store.dirty_session_rollups().expect("sessions"); + assert_eq!(sessions[0].title.as_deref(), Some(title)); +} + +#[test] +fn rebuild_keeps_unchanged_sessions_synced() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-rebuild"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 8, 9, 0, 0) + .single() + .expect("start"); + let mut event = test_store_event(&source, start, "rebuild"); + stamp_session(&mut event, "rebuild-session"); + store.insert_event(&event).expect("insert"); + let ids = store + .dirty_session_rollups() + .expect("dirty") + .into_iter() + .map(|session| session.session_id) + .collect::>(); + store.mark_session_rollups_synced(&ids).expect("synced"); + + store.rebuild_session_rollups().expect("rebuild"); + assert!(store.dirty_session_rollups().expect("clean").is_empty()); +} diff --git a/crates/statsai/tests/cli_surface/schema-sync-batch.json b/crates/statsai/tests/cli_surface/schema-sync-batch.json index f9fb267..cb9e7b4 100644 --- a/crates/statsai/tests/cli_surface/schema-sync-batch.json +++ b/crates/statsai/tests/cli_surface/schema-sync-batch.json @@ -1541,7 +1541,6 @@ "required": [ "cost", "device_id", - "duration_seconds", "ended_at", "models", "provider", @@ -1582,8 +1581,11 @@ "type": "string" }, "duration_seconds": { - "description": "Always a non-negative integer on the wire. Unknown durations serialize as 0 because `session_rollup.v1` rejects null.", - "type": "integer", + "description": "`None` when the session recorded only a start. Unknown durations go out as null; ingest stores them without an end so they stay out of medians.", + "type": [ + "integer", + "null" + ], "format": "uint64", "minimum": 0.0 }, diff --git a/docs/sync-contract.md b/docs/sync-contract.md index 7d8fd2e..3ff97f8 100644 --- a/docs/sync-contract.md +++ b/docs/sync-contract.md @@ -27,7 +27,10 @@ The same batch schema carries opt-in `sessions` rows. Each row is `session_rollup.v1` with a closed key set: hashed session id, provider, source, account, time range, duration, usage, requests, cost, models, message counts, project, bounded title, and `title_source` (`event`, `task_span`, or -`archive`). Prompts, responses, paths, and raw provider session ids stay on +`archive`). `duration_seconds` is null when a session recorded only a start; +the backend must accept null before a collector sends it. Events keyed by their +own provider record, such as Cursor's usage-export rows, belong to no session +and are not sent. Prompts, responses, paths, and raw provider session ids stay on the device. Session retirement is not implied by `sync_batch.v6`. A snapshot retires hosted sessions only when it includes `session_rollup_ids`. An empty array prunes them. Omitting the key leaves hosted sessions in place, which is From 8df85715552fad12ccdb756c62b85bc7f7307ef1 Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Sat, 26 Sep 2026 13:36:39 +0300 Subject: [PATCH 06/16] Keep provider session names and resend sessions after the rebuild Session titles the provider put on its events, such as Codex's "Review uncommitted changes", were dropped by the genericness rules written for prompt-derived task titles. Event titles now drop only placeholders and text that looks like a secret or a locator; task-span and archive titles keep the stricter rules, locally and at sync. The v2 session backfill also forgets every target's session acknowledgements, so the first sync after it resends all sessions. Hosted session tables are rebuilt empty for this release; where a hosted row survived, the resend is an upsert that writes nothing. --- crates/statsai-core/src/tasks/titles/mod.rs | 23 +++++ crates/statsai-core/src/tests/types.rs | 15 ++++ crates/statsai-core/src/types/sync.rs | 8 +- crates/statsai-store/src/rollups/sessions.rs | 51 +++++++---- .../src/tests/rollups/sessions.rs | 90 ++++++++++++++++++- 5 files changed, 166 insertions(+), 21 deletions(-) diff --git a/crates/statsai-core/src/tasks/titles/mod.rs b/crates/statsai-core/src/tasks/titles/mod.rs index ff399c2..9e9c397 100644 --- a/crates/statsai-core/src/tasks/titles/mod.rs +++ b/crates/statsai-core/src/tasks/titles/mod.rs @@ -96,6 +96,29 @@ pub fn task_title_signal_score(value: Option<&str>) -> i32 { score } +/// Whether a session name the provider itself assigned is unfit to show. +/// +/// Provider names are shown as the provider shows them, so "Review +/// uncommitted changes" or "Say hi" stay. The genericness rules in +/// [`task_title_is_generic`] rank prompt-derived task titles; here only a +/// placeholder or text that looks like a secret or a locator is dropped. +#[must_use] +pub fn provider_session_title_is_unusable(value: Option<&str>) -> bool { + let Some(raw) = value else { + return true; + }; + if looks_like_sensitive_locator_dump(raw) { + return true; + } + let Some(value) = clean_task_text(raw) else { + return true; + }; + let value = polish_task_title_candidate(&value); + value.is_empty() + || looks_like_sensitive_locator_dump(&value.to_ascii_lowercase()) + || looks_like_provider_placeholder_title(&value) +} + #[must_use] pub fn task_title_is_generic(value: Option<&str>) -> bool { let Some(raw) = value else { diff --git a/crates/statsai-core/src/tests/types.rs b/crates/statsai-core/src/tests/types.rs index 72cbd80..d6e06d6 100644 --- a/crates/statsai-core/src/tests/types.rs +++ b/crates/statsai-core/src/tests/types.rs @@ -659,6 +659,21 @@ fn sanitize_session_rollup_repairs_values_ingest_rejects() { assert_eq!(sanitized.ended_at, sanitized.started_at); assert_eq!(sanitized.duration_seconds, Some(0)); + let mut provider_named = sample_session_rollup(started); + provider_named.title = Some("Review uncommitted changes".to_string()); + provider_named.title_source = Some(SessionTitleSource::Event); + let sanitized = sanitize_session_rollup_for_sync(provider_named); + assert_eq!( + sanitized.title.as_deref(), + Some("Review uncommitted changes") + ); + let mut prompt_derived = sample_session_rollup(started); + prompt_derived.title = Some("Review uncommitted changes".to_string()); + prompt_derived.title_source = Some(SessionTitleSource::TaskSpan); + let sanitized = sanitize_session_rollup_for_sync(prompt_derived); + assert_eq!(sanitized.title, None); + assert_eq!(sanitized.title_source, None); + let mut unknown = sample_session_rollup(started); unknown.duration_seconds = None; let sanitized = sanitize_session_rollup_for_sync(unknown); diff --git a/crates/statsai-core/src/types/sync.rs b/crates/statsai-core/src/types/sync.rs index 5b12873..69f91ca 100644 --- a/crates/statsai-core/src/types/sync.rs +++ b/crates/statsai-core/src/types/sync.rs @@ -366,7 +366,13 @@ fn clamp_session_title(rollup: &mut SessionRollupV1) { .title .as_deref() .and_then(|label| sync_label(label, SYNC_LABEL_UTF16_MAX)) - .filter(|label| !crate::task_title_is_generic(Some(label))); + .filter(|label| match rollup.title_source { + // The provider named the session; keep its name unless unsafe. + Some(crate::SessionTitleSource::Event) => { + !crate::provider_session_title_is_unusable(Some(label)) + } + _ => !crate::task_title_is_generic(Some(label)), + }); if title.is_none() { rollup.title_source = None; } diff --git a/crates/statsai-store/src/rollups/sessions.rs b/crates/statsai-store/src/rollups/sessions.rs index 5d02fbb..e789de4 100644 --- a/crates/statsai-store/src/rollups/sessions.rs +++ b/crates/statsai-store/src/rollups/sessions.rs @@ -6,14 +6,15 @@ use anyhow::Result; use chrono::{DateTime, Utc}; use rusqlite::{params, OptionalExtension}; use statsai_core::{ - hash_text, project_has_stable_identity, task_title_is_generic, Confidence, CostAccumulator, - CostInfo, ModelInfo, SessionRollupV1, SessionTitleSource, SummaryModelUsage, UsageCounts, - UsageEvent, SESSION_ROLLUP_SCHEMA_VERSION, + hash_text, project_has_stable_identity, provider_session_title_is_unusable, + task_title_is_generic, Confidence, CostAccumulator, CostInfo, ModelInfo, SessionRollupV1, + SessionTitleSource, SummaryModelUsage, UsageCounts, UsageEvent, SESSION_ROLLUP_SCHEMA_VERSION, }; use std::collections::{BTreeMap, BTreeSet}; // v2 drops record-keyed rows, leaves single-instant durations unknown, and -// joins Codex rollout-stem span titles. +// joins Codex rollout-stem span titles. Hosted session tables were rebuilt +// for the same release, so the v2 rebuild resends every session. const SESSION_ROLLUPS_BACKFILL_KEY: &str = "session_rollups_backfilled_v2"; /// Matches the adapters' `provider_record_usage_event.v1` event key version. const PROVIDER_RECORD_ID_PREFIX: &str = "provider_record_usage_event.v1:"; @@ -59,6 +60,14 @@ impl Store { return Ok(()); } self.rebuild_session_rollups()?; + // Hosted session tables were rebuilt empty for this release. Resends + // follow each target's acknowledgements, so forgetting them makes the + // next sync send every session; where a hosted row survived, that + // upsert writes nothing. + self.conn.execute( + "DELETE FROM entity_sync_state WHERE entity_kind = 'session_rollup'", + [], + )?; self.set_metadata_value(SESSION_ROLLUPS_BACKFILL_KEY, "1")?; Ok(()) } @@ -76,8 +85,10 @@ impl Store { .push(event); } self.with_immediate_transaction(|| { - // Rows whose content is unchanged keep their sync state, so a - // rebuild after a builder change resends only what it changed. + // Every session is resent after a rebuild, since the hosted table + // may have been rebuilt with it. Unchanged rows keep their payload + // and updated_at, so where the hosted row survived, the resend is + // a no-op upsert that writes nothing. let existing = self.session_rollup_payload_hashes()?; for session_id in existing.keys() { if !grouped.contains_key(session_id) { @@ -98,6 +109,10 @@ impl Store { let rollup = build_session_rollup(events, &titles, updated_at); let payload_hash = session_rollup_content_hash(&rollup)?; if existing.get(session_id) == Some(&payload_hash) { + self.conn.execute( + "UPDATE session_rollups SET dirty = 1 WHERE session_id = ?1", + params![session_id], + )?; continue; } self.write_session_rollup(&rollup, &payload_hash, 1)?; @@ -649,11 +664,9 @@ fn resolve_title( events: &[UsageEvent], titles: &BTreeMap, ) -> (Option, Option) { - if let Some(title) = events - .iter() - .rev() - .find_map(|event| acceptable_session_title(event.session.title.as_deref())) - { + if let Some(title) = events.iter().rev().find_map(|event| { + acceptable_session_title(event.session.title.as_deref(), SessionTitleSource::Event) + }) { return (Some(title), Some(SessionTitleSource::Event)); } for event in events.iter().rev() { @@ -698,13 +711,15 @@ fn codex_rollout_session_uuid(raw_id: &str) -> Option<&str> { is_uuid.then_some(uuid) } -fn acceptable_session_title(value: Option<&str>) -> Option { +/// Names the provider put on its events are kept unless unsafe. Task-span and +/// archive titles can come from prompts, so they must not be generic. +fn acceptable_session_title(value: Option<&str>, source: SessionTitleSource) -> Option { let trimmed = value?.trim(); - if trimmed.is_empty() || task_title_is_generic(Some(trimmed)) { - None - } else { - Some(trimmed.to_string()) - } + let rejected = match source { + SessionTitleSource::Event => provider_session_title_is_unusable(Some(trimmed)), + _ => task_title_is_generic(Some(trimmed)), + }; + (!trimmed.is_empty() && !rejected).then(|| trimmed.to_string()) } fn remember_hashed_title( @@ -714,7 +729,7 @@ fn remember_hashed_title( title: Option<&str>, source: SessionTitleSource, ) { - let Some(title) = acceptable_session_title(title) else { + let Some(title) = acceptable_session_title(title, source) else { return; }; let hash = hash_text(raw_id); diff --git a/crates/statsai-store/src/tests/rollups/sessions.rs b/crates/statsai-store/src/tests/rollups/sessions.rs index 6429d60..fedea2f 100644 --- a/crates/statsai-store/src/tests/rollups/sessions.rs +++ b/crates/statsai-store/src/tests/rollups/sessions.rs @@ -510,7 +510,7 @@ fn codex_rollout_stem_span_titles_join_uuid_sessions() { } #[test] -fn rebuild_keeps_unchanged_sessions_synced() { +fn rebuild_resends_unchanged_sessions_without_restamping() { let store = Store::in_memory().expect("store"); let source = SourceLocation::local_adapter( "codex", @@ -533,8 +533,94 @@ fn rebuild_keeps_unchanged_sessions_synced() { .into_iter() .map(|session| session.session_id) .collect::>(); + let before = store.dirty_session_rollups().expect("before"); store.mark_session_rollups_synced(&ids).expect("synced"); store.rebuild_session_rollups().expect("rebuild"); - assert!(store.dirty_session_rollups().expect("clean").is_empty()); + let resent = store.dirty_session_rollups().expect("resent"); + assert_eq!(resent, before); +} + +#[test] +fn provider_named_sessions_keep_plain_names_but_drop_placeholders() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-provider-titles"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 9, 9, 0, 0) + .single() + .expect("start"); + let plain = "Review uncommitted changes"; + assert!(task_title_is_generic(Some(plain))); + let mut review = test_store_event(&source, start, "review"); + stamp_session(&mut review, "review-session"); + review.session.title = Some(plain.to_string()); + store.insert_event(&review).expect("review"); + + let mut placeholder = test_store_event(&source, start, "placeholder"); + stamp_session(&mut placeholder, "placeholder-session"); + placeholder.session.title = Some("New session - 2026-04-30T16:41:41.413Z".to_string()); + store.insert_event(&placeholder).expect("placeholder"); + + let sessions = store.dirty_session_rollups().expect("sessions"); + let title = |event: &UsageEvent| { + sessions + .iter() + .find(|session| session.session_id == event.session.session_id) + .expect("session") + .title + .clone() + }; + assert_eq!(title(&review).as_deref(), Some(plain)); + assert_eq!(title(&placeholder), None); +} + +#[test] +fn session_backfill_resends_every_session_to_each_target() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-backfill-resend"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 10, 9, 0, 0) + .single() + .expect("start"); + let mut event = test_store_event(&source, start, "backfill"); + stamp_session(&mut event, "backfill-session"); + store.insert_event(&event).expect("insert"); + let sessions = store.all_session_rollups().expect("sessions"); + store + .record_session_rollups_synced("http", "https://api.example.test", &sessions) + .expect("record"); + assert!(store + .pending_session_rollups_for_sync("http", "https://api.example.test", &sessions) + .expect("pending") + .is_empty()); + + store + .conn + .execute( + "DELETE FROM local_metadata WHERE key LIKE 'session_rollups_backfilled_%'", + [], + ) + .expect("forget backfill"); + store + .backfill_session_rollups_if_needed() + .expect("backfill"); + + let pending = store + .pending_session_rollups_for_sync("http", "https://api.example.test", &sessions) + .expect("pending after backfill"); + assert_eq!(pending, sessions); } From 51a6e934527440bf0d18705054755b9c530413c8 Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Sat, 26 Sep 2026 14:28:13 +0300 Subject: [PATCH 07/16] Name Codex sub-agents after their parent and keep provider names intact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex never names its sub-agents. A spawned agent records its parent thread and a guardian review carries the parent's id as session_id, so their events now take the parent's thread name marked with the agent's nickname or kind, such as "Fix parser bug · guardian". Task titles keep their own rules. Claude Code titles were applied only to events read from the transcript that named the session. A resumed session's copied messages dedupe onto the earlier transcript, so titles now apply once the whole source is read. Provider-assigned names were cleaned by the prompt rules, which drop a name like "Design tool/plugin" as a file reference. They now only collapse whitespace and truncate, and default names such as "New session" count as no title. --- crates/statsai-adapters/src/claude/mod.rs | 10 ++- crates/statsai-adapters/src/claude/parse.rs | 69 +++++++++++------- .../src/claude/tests/parse.rs | 36 +++++++++ .../statsai-adapters/src/codex/parse/mod.rs | 28 +++---- .../src/codex/parse/session.rs | 34 +++++++++ .../src/codex/tests/parse/session.rs | 73 +++++++++++++++++++ .../statsai-core/src/tasks/normalize/mod.rs | 18 +++++ crates/statsai-core/src/tasks/titles/mod.rs | 32 +++++--- crates/statsai-core/src/tests/types.rs | 8 ++ .../src/tests/rollups/sessions.rs | 6 ++ 10 files changed, 260 insertions(+), 54 deletions(-) diff --git a/crates/statsai-adapters/src/claude/mod.rs b/crates/statsai-adapters/src/claude/mod.rs index 308420a..98f8f5c 100644 --- a/crates/statsai-adapters/src/claude/mod.rs +++ b/crates/statsai-adapters/src/claude/mod.rs @@ -205,6 +205,7 @@ pub(crate) fn scan_claude_source( let event_files = claude_jsonl_candidates(&projects, &cache_namespaces)?; let mut scanned_event_cache_keys = HashSet::new(); let mut seen = EventDedupIndex::new(); + let mut session_titles = ClaudeSessionTitles::default(); { let mut ctx = FileParseContext { adapter, @@ -220,9 +221,16 @@ pub(crate) fn scan_claude_source( } ctx.scan.diagnostics.files_scanned += 1; scanned_event_cache_keys.insert(candidate.cache_key.clone()); - parse_claude_file(&mut ctx, &projects, &session_projects, &candidate.path)?; + parse_claude_file( + &mut ctx, + &projects, + &session_projects, + &mut session_titles, + &candidate.path, + )?; } } + session_titles.apply(&mut scan.events); if let Some(candidate) = claude_stats_cache_candidate(&root, &cache_namespaces) { if options.should_scan(&candidate.cache_key) { diff --git a/crates/statsai-adapters/src/claude/parse.rs b/crates/statsai-adapters/src/claude/parse.rs index 9f49c7b..6d09111 100644 --- a/crates/statsai-adapters/src/claude/parse.rs +++ b/crates/statsai-adapters/src/claude/parse.rs @@ -17,7 +17,7 @@ use anyhow::{Context, Result}; use chrono::Utc; use serde_json::Value; use statsai_core::{ - canonical_display, expand_home_path, hash_text, summarize_task_text, summary_id, Confidence, + canonical_display, expand_home_path, hash_text, provider_session_name, summary_id, Confidence, EventSource, IdentitySource, ParseEvidence, ProjectInfo, SourceKind, SourceLocation, SummaryMetadata, UsageCounts, UsageSummary, USAGE_SUMMARY_SCHEMA_VERSION, }; @@ -31,6 +31,7 @@ pub(crate) fn parse_claude_file( ctx: &mut FileParseContext<'_, ClaudeCodeAdapter>, projects: &Path, session_projects: &HashMap, + session_titles: &mut ClaudeSessionTitles, path: &Path, ) -> Result<()> { let file = File::open(path).with_context(|| format!("read {}", path.display()))?; @@ -48,8 +49,6 @@ pub(crate) fn parse_claude_file( // Later snapshots of one provider record replace the event. User lines // counted on the first snapshot have to ride along with that replacement. let mut last_usage_users = HashMap::::new(); - let mut session_titles = HashMap::::new(); - let events_start = ctx.scan.events.len(); loop { let line_status = read_bounded_jsonl_line(&mut reader, &mut line_bytes, MAX_JSONL_RECORD_BYTES)?; @@ -85,12 +84,7 @@ pub(crate) fn parse_claude_file( } let session_raw = claude_session_raw(&value, path); if let Some((rank, title)) = claude_session_title_from_value(&value) { - let replaces = session_titles - .get(&session_raw) - .is_none_or(|(current, _)| rank >= *current); - if replaces { - session_titles.insert(session_raw, (rank, title)); - } + session_titles.remember(session_raw, rank, title); continue; } let Some(usage_value) = value @@ -190,14 +184,48 @@ pub(crate) fn parse_claude_file( push_deduped(ctx.scan, ctx.seen, event, duplicate_selection); } - // Title records can follow the usage they name, so titles land once the - // whole file has been read. - if !session_titles.is_empty() { - let titles_by_hash = session_titles + let activity_started_at = std::time::Instant::now(); + activity.finish( + ctx.scan, + &ctx.options.device_id, + ctx.source, + fallback_timestamp, + ); + ctx.scan.diagnostics.activity_extract_ms += activity_started_at.elapsed().as_millis() as u64; + + Ok(()) +} + +/// Session names found across one source's transcripts. +/// +/// A resumed or forked session copies earlier messages into a new transcript, +/// and those events dedupe onto the positions of the file that first held +/// them. Titles are therefore applied once the whole source is read, by session +/// hash, rather than to the events of the file that named the session. +#[derive(Default)] +pub(crate) struct ClaudeSessionTitles(HashMap); + +impl ClaudeSessionTitles { + fn remember(&mut self, session_raw: String, rank: u8, title: String) { + let replaces = self + .0 + .get(&session_raw) + .is_none_or(|(current, _)| rank >= *current); + if replaces { + self.0.insert(session_raw, (rank, title)); + } + } + + pub(crate) fn apply(self, events: &mut [statsai_core::UsageEvent]) { + if self.0.is_empty() { + return; + } + let titles_by_hash = self + .0 .into_iter() .map(|(session_raw, (_, title))| (hash_text(&session_raw), title)) .collect::>(); - for event in &mut ctx.scan.events[events_start..] { + for event in events { if let Some(title) = event .session .local_session_id_hash @@ -208,17 +236,6 @@ pub(crate) fn parse_claude_file( } } } - - let activity_started_at = std::time::Instant::now(); - activity.finish( - ctx.scan, - &ctx.options.device_id, - ctx.source, - fallback_timestamp, - ); - ctx.scan.diagnostics.activity_extract_ms += activity_started_at.elapsed().as_millis() as u64; - - Ok(()) } /// Claude Code names a session with `custom-title` records, which a rename @@ -230,7 +247,7 @@ fn claude_session_title_from_value(value: &Value) -> Option<(u8, String)> { "ai-title" => (0, "aiTitle"), _ => return None, }; - let title = summarize_task_text(value.get(key).and_then(Value::as_str), 90)?; + let title = provider_session_name(value.get(key).and_then(Value::as_str), 90)?; Some((rank, title)) } diff --git a/crates/statsai-adapters/src/claude/tests/parse.rs b/crates/statsai-adapters/src/claude/tests/parse.rs index 3d0bfd2..f0a8946 100644 --- a/crates/statsai-adapters/src/claude/tests/parse.rs +++ b/crates/statsai-adapters/src/claude/tests/parse.rs @@ -1215,3 +1215,39 @@ fn claude_session_titles_name_every_event_in_the_session() { ); assert_eq!(title_for("untitled"), None); } + +#[test] +fn claude_session_titles_reach_events_read_from_other_transcripts() { + let dir = tempfile::tempdir().expect("tempdir"); + let projects = dir.path().join("projects"); + std::fs::create_dir_all(&projects).expect("projects"); + // A resumed session's copied messages dedupe onto the transcript that + // first held them, while the title is written to the resumed transcript. + let mut original = File::create(projects.join("original.jsonl")).expect("original"); + writeln!( + original, + r#"{{"timestamp":"2026-05-01T00:00:00Z","sessionId":"resumed","message":{{"id":"m1","usage":{{"input_tokens":1,"output_tokens":2}}}}}}"# + ) + .expect("write original"); + let mut resumed = File::create(projects.join("resumed.jsonl")).expect("resumed"); + writeln!( + resumed, + r#"{{"type":"custom-title","customTitle":"Design tool/plugin","sessionId":"resumed"}}"# + ) + .expect("write resumed"); + + let source = SourceLocation::local_adapter( + CLAUDE_CODE_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let scan = scan_claude_source(&ClaudeCodeAdapter, &source, &options()).expect("scan"); + + assert_eq!(scan.events.len(), 1); + assert_eq!( + scan.events[0].session.title.as_deref(), + Some("Design tool/plugin") + ); +} diff --git a/crates/statsai-adapters/src/codex/parse/mod.rs b/crates/statsai-adapters/src/codex/parse/mod.rs index ad109c1..c068d43 100644 --- a/crates/statsai-adapters/src/codex/parse/mod.rs +++ b/crates/statsai-adapters/src/codex/parse/mod.rs @@ -39,6 +39,9 @@ pub(crate) fn parse_codex_file( let mut current_model_is_fallback = false; let mut current_project: Option = None; let mut current_title: Option = None; + // Events only: a sub-agent borrows its parent thread's name, but task + // titles keep their own prompt-based rules. + let mut subagent_title: Option = None; let mut current_thread_id: Option = None; // Seeded from the file path, replaced by the session's own id as soon as // the `session_meta` line declares one. The embedded id is the same UUID @@ -343,20 +346,13 @@ pub(crate) fn parse_codex_file( if collect_tasks { current_thread_id = declared_session_id.clone(); } - { - let session_id = declared_session_id.or_else(|| Some(session_raw.clone())); - current_title = value - .pointer("/payload/thread_name") - .and_then(Value::as_str) - .map(ToOwned::to_owned) - .or_else(|| { - session_id - .as_ref() - .and_then(|session_id| thread_titles.get(session_id)) - .cloned() - }) - .or_else(|| thread_titles.get(&session_raw).cloned()); - } + let session_id = declared_session_id.unwrap_or_else(|| session_raw.clone()); + current_title = value + .pointer("/payload/thread_name") + .and_then(Value::as_str) + .map(ToOwned::to_owned) + .or_else(|| thread_titles.get(&session_id).cloned()); + subagent_title = codex_subagent_session_title(&value, &session_id, thread_titles); current_project = codex_project_context_from_value(&value, &mut project_cache); continue; } @@ -734,7 +730,7 @@ pub(crate) fn parse_codex_file( dedupe_salt: None, }, ); - event.session.title = turn.title.clone(); + event.session.title = turn.title.clone().or_else(|| subagent_title.clone()); let mut linked_quota_lines = turn.usage_lines.clone(); linked_quota_lines.extend_from_slice(&turn.quota_lines); if record.usage.is_some() { @@ -931,7 +927,7 @@ pub(crate) fn parse_codex_file( if consumed_usage_lines.contains(&record.line_number) { continue; } - let session_title = record.session_title; + let session_title = record.session_title.or_else(|| subagent_title.clone()); let mut event = usage_event( ctx.adapter, ctx.source, diff --git a/crates/statsai-adapters/src/codex/parse/session.rs b/crates/statsai-adapters/src/codex/parse/session.rs index 6111df6..1a47ff0 100644 --- a/crates/statsai-adapters/src/codex/parse/session.rs +++ b/crates/statsai-adapters/src/codex/parse/session.rs @@ -36,6 +36,40 @@ pub(crate) fn load_codex_thread_titles(root: &Path) -> HashMap { titles } +/// Codex never names its sub-agents. A spawned agent records its parent +/// thread, and a guardian review carries the parent's id as `session_id`, so +/// both borrow the parent's name, marked with the agent's nickname or kind. +pub(crate) fn codex_subagent_session_title( + value: &Value, + session_id: &str, + thread_titles: &HashMap, +) -> Option { + let payload = value.get("payload")?; + let subagent = payload.pointer("/source/subagent")?; + let (parent, label) = if let Some(spawn) = subagent.get("thread_spawn") { + let label = ["agent_nickname", "agent_role"] + .iter() + .find_map(|key| spawn.get(*key).and_then(Value::as_str)) + .unwrap_or("sub-agent"); + ( + spawn.get("parent_thread_id").and_then(Value::as_str)?, + label, + ) + } else { + let parent = payload + .get("session_id") + .and_then(Value::as_str) + .filter(|parent| *parent != session_id)?; + let label = subagent + .get("other") + .and_then(Value::as_str) + .unwrap_or("sub-agent"); + (parent, label) + }; + let parent_title = thread_titles.get(parent)?; + statsai_core::provider_session_name(Some(&format!("{parent_title} · {label}")), 90) +} + pub(crate) fn codex_project_context_from_value( value: &Value, cache: &mut ProjectContextCache, diff --git a/crates/statsai-adapters/src/codex/tests/parse/session.rs b/crates/statsai-adapters/src/codex/tests/parse/session.rs index 82b5d29..ff219f5 100644 --- a/crates/statsai-adapters/src/codex/tests/parse/session.rs +++ b/crates/statsai-adapters/src/codex/tests/parse/session.rs @@ -405,3 +405,76 @@ fn codex_task_spans_capture_thread_id_from_session_meta() { .iter() .all(|event| event.session.title.as_deref() == Some("Fix parser bug"))); } + +#[test] +fn codex_subagents_borrow_their_parent_thread_name() { + let dir = tempfile::tempdir().expect("tempdir"); + let codex_root = dir.path().join("codex"); + let sessions = codex_root.join("sessions"); + std::fs::create_dir_all(&sessions).expect("sessions"); + std::fs::write( + codex_root.join("session_index.jsonl"), + "{\"id\":\"parent-1\",\"thread_name\":\"Fix parser bug\"}\n", + ) + .expect("session index"); + for (index, (file, meta)) in [ + ( + "spawned.jsonl", + r#"{"timestamp":"2026-06-01T08:00:00Z","type":"session_meta","payload":{"id":"child-spawned","source":{"subagent":{"thread_spawn":{"parent_thread_id":"parent-1","depth":1,"agent_nickname":"Popper","agent_role":null}}}}}"#, + ), + ( + "guardian.jsonl", + r#"{"timestamp":"2026-06-01T08:00:00Z","type":"session_meta","payload":{"id":"child-guardian","session_id":"parent-1","source":{"subagent":{"other":"guardian"}}}}"#, + ), + ( + "orphan.jsonl", + r#"{"timestamp":"2026-06-01T08:00:00Z","type":"session_meta","payload":{"id":"child-orphan","source":{"subagent":{"other":"guardian"}}}}"#, + ), + ] + .into_iter() + .enumerate() + { + // Distinct usage per rollout, or the path-independent dedupe keeps one. + let input = 60 + index; + let total = 120 + index; + let usage = format!( + r#"{{"timestamp":"2026-06-01T08:00:03Z","type":"event_msg","payload":{{"type":"token_count","info":{{"last_token_usage":{{"input_tokens":{input},"cached_input_tokens":20,"output_tokens":30,"reasoning_output_tokens":10,"total_tokens":{total}}},"total_token_usage":{{"input_tokens":{input},"cached_input_tokens":20,"output_tokens":30,"reasoning_output_tokens":10,"total_tokens":{total}}}}}}}}}"# + ); + let mut handle = File::create(sessions.join(file)).expect("rollout"); + writeln!(handle, "{meta}").expect("meta"); + writeln!( + handle, + r#"{{"timestamp":"2026-06-01T08:00:00Z","type":"turn_context","payload":{{"model":"gpt-5"}}}}"# + ) + .expect("context"); + writeln!(handle, "{usage}").expect("usage"); + } + + let source = SourceLocation::local_adapter( + CODEX_PROVIDER, + "test", + "0", + &codex_root, + LocationOrigin::Configured, + ); + let scan = scan_codex_source(&CodexAdapter, &source, &options_without_tasks()).expect("scan"); + let title_for = |raw: &str| { + let hash = statsai_core::hash_text(raw); + scan.events + .iter() + .find(|event| event.session.local_session_id_hash.as_deref() == Some(hash.as_str())) + .expect("event") + .session + .title + .clone() + }; + assert_eq!( + title_for("child-spawned").as_deref(), + Some("Fix parser bug · Popper") + ); + assert_eq!( + title_for("child-guardian").as_deref(), + Some("Fix parser bug · guardian") + ); + assert_eq!(title_for("child-orphan"), None); +} diff --git a/crates/statsai-core/src/tasks/normalize/mod.rs b/crates/statsai-core/src/tasks/normalize/mod.rs index 9579ac2..7f6e6b6 100644 --- a/crates/statsai-core/src/tasks/normalize/mod.rs +++ b/crates/statsai-core/src/tasks/normalize/mod.rs @@ -49,6 +49,24 @@ pub fn summarize_task_text(value: Option<&str>, width: usize) -> Option truncate_task_text(clean_task_text(value?)?, width) } +/// Normalizes a session name the provider assigned: control characters and +/// runs of whitespace collapse to single spaces and the result is truncated. +/// +/// [`summarize_task_text`] cleans prompts, and its line filters drop a name +/// like "Design tool/plugin" as a file reference. +#[must_use] +pub fn provider_session_name(value: Option<&str>, width: usize) -> Option { + let compact = value? + .split(|character: char| character.is_whitespace() || character.is_control()) + .filter(|part| !part.is_empty()) + .collect::>() + .join(" "); + if compact.is_empty() { + return None; + } + truncate_task_text(compact, width) +} + #[must_use] pub fn choose_best_task_title<'a>( primary: Option<&'a str>, diff --git a/crates/statsai-core/src/tasks/titles/mod.rs b/crates/statsai-core/src/tasks/titles/mod.rs index 9e9c397..8341605 100644 --- a/crates/statsai-core/src/tasks/titles/mod.rs +++ b/crates/statsai-core/src/tasks/titles/mod.rs @@ -104,19 +104,29 @@ pub fn task_title_signal_score(value: Option<&str>) -> i32 { /// placeholder or text that looks like a secret or a locator is dropped. #[must_use] pub fn provider_session_title_is_unusable(value: Option<&str>) -> bool { - let Some(raw) = value else { + let Some(name) = super::normalize::provider_session_name(value, usize::MAX) else { return true; }; - if looks_like_sensitive_locator_dump(raw) { - return true; - } - let Some(value) = clean_task_text(raw) else { - return true; - }; - let value = polish_task_title_candidate(&value); - value.is_empty() - || looks_like_sensitive_locator_dump(&value.to_ascii_lowercase()) - || looks_like_provider_placeholder_title(&value) + looks_like_sensitive_locator_dump(&name) + || looks_like_provider_placeholder_title(&name) + || looks_like_session_placeholder_name(&name) +} + +/// Default names a tool gives a session before anything names it, such as +/// OpenCode's "New session - 2026-04-30T16:41:41.413Z". +const SESSION_PLACEHOLDER_NAMES: &[&str] = &[ + "new session", + "new chat", + "new conversation", + "new thread", + "untitled", + "untitled session", +]; + +fn looks_like_session_placeholder_name(name: &str) -> bool { + let lowercase = name.to_ascii_lowercase(); + let head = lowercase.split(" - ").next().unwrap_or_default().trim(); + SESSION_PLACEHOLDER_NAMES.contains(&head) } #[must_use] diff --git a/crates/statsai-core/src/tests/types.rs b/crates/statsai-core/src/tests/types.rs index d6e06d6..c7a1051 100644 --- a/crates/statsai-core/src/tests/types.rs +++ b/crates/statsai-core/src/tests/types.rs @@ -667,6 +667,14 @@ fn sanitize_session_rollup_repairs_values_ingest_rejects() { sanitized.title.as_deref(), Some("Review uncommitted changes") ); + let mut slashed = sample_session_rollup(started); + slashed.title = Some("Claude Design tool/plugin".to_string()); + slashed.title_source = Some(SessionTitleSource::Event); + let sanitized = sanitize_session_rollup_for_sync(slashed); + assert_eq!( + sanitized.title.as_deref(), + Some("Claude Design tool/plugin") + ); let mut prompt_derived = sample_session_rollup(started); prompt_derived.title = Some("Review uncommitted changes".to_string()); prompt_derived.title_source = Some(SessionTitleSource::TaskSpan); diff --git a/crates/statsai-store/src/tests/rollups/sessions.rs b/crates/statsai-store/src/tests/rollups/sessions.rs index fedea2f..6c399e7 100644 --- a/crates/statsai-store/src/tests/rollups/sessions.rs +++ b/crates/statsai-store/src/tests/rollups/sessions.rs @@ -568,6 +568,11 @@ fn provider_named_sessions_keep_plain_names_but_drop_placeholders() { placeholder.session.title = Some("New session - 2026-04-30T16:41:41.413Z".to_string()); store.insert_event(&placeholder).expect("placeholder"); + let mut bare = test_store_event(&source, start, "bare-placeholder"); + stamp_session(&mut bare, "bare-placeholder-session"); + bare.session.title = Some("New session".to_string()); + store.insert_event(&bare).expect("bare placeholder"); + let sessions = store.dirty_session_rollups().expect("sessions"); let title = |event: &UsageEvent| { sessions @@ -579,6 +584,7 @@ fn provider_named_sessions_keep_plain_names_but_drop_placeholders() { }; assert_eq!(title(&review).as_deref(), Some(plain)); assert_eq!(title(&placeholder), None); + assert_eq!(title(&bare), None); } #[test] From a4b2176699b767e3d4157b7087d59ba95d1be3dd Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Sat, 26 Sep 2026 15:38:40 +0300 Subject: [PATCH 08/16] Keep only sessions with a project or a title A session with neither cannot be told apart from any other. Cursor cloud agents are the case: the usage export names the agent by id only and records no project. The store no longer keeps or syncs them. The v2 session backfill now clears each target's acknowledged payload hash instead of deleting the acknowledgement, so every session is still resent while a session the rebuild dropped is retired by the next snapshot. --- README.md | 3 +- crates/statsai-store/src/rollups/sessions.rs | 65 +++++++++++++------ .../src/tests/rollups/sessions.rs | 56 ++++++++++++++++ docs/sync-contract.md | 2 +- 4 files changed, 104 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index ebe6128..1b6bfc4 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,8 @@ excerpts. Hosted session sync is separate and off by default. hashed session ids, token and request totals, duration, message counts, and a bounded title. Titles come from the provider's own session name (Codex thread names, Claude Code and OpenCode session titles) or a local task title. Cursor's -usage export names no session, so its local rows are not sessions. +usage export names no session, so its local rows are not sessions, and a session +with neither a project nor a title is not kept. `--exclude-sessions` stops sending them and leaves sessions already hosted in place. You can inspect the exact sync contract in [`docs/sync-contract.md`](docs/sync-contract.md) and verify the resolved sync diff --git a/crates/statsai-store/src/rollups/sessions.rs b/crates/statsai-store/src/rollups/sessions.rs index e789de4..08f60cb 100644 --- a/crates/statsai-store/src/rollups/sessions.rs +++ b/crates/statsai-store/src/rollups/sessions.rs @@ -12,9 +12,10 @@ use statsai_core::{ }; use std::collections::{BTreeMap, BTreeSet}; -// v2 drops record-keyed rows, leaves single-instant durations unknown, and -// joins Codex rollout-stem span titles. Hosted session tables were rebuilt -// for the same release, so the v2 rebuild resends every session. +// v2 drops record-keyed rows and sessions with neither project nor title, +// leaves single-instant durations unknown, and joins Codex rollout-stem span +// titles. Hosted session tables were rebuilt for the same release, so the v2 +// rebuild resends every session. const SESSION_ROLLUPS_BACKFILL_KEY: &str = "session_rollups_backfilled_v2"; /// Matches the adapters' `provider_record_usage_event.v1` event key version. const PROVIDER_RECORD_ID_PREFIX: &str = "provider_record_usage_event.v1:"; @@ -61,11 +62,12 @@ impl Store { } self.rebuild_session_rollups()?; // Hosted session tables were rebuilt empty for this release. Resends - // follow each target's acknowledgements, so forgetting them makes the - // next sync send every session; where a hosted row survived, that - // upsert writes nothing. + // follow each target's acknowledged payload hash, so clearing the hash + // makes the next sync send every session; where a hosted row survived, + // that upsert writes nothing. The acknowledgements themselves stay, so + // a session the rebuild dropped is still retired by the next snapshot. self.conn.execute( - "DELETE FROM entity_sync_state WHERE entity_kind = 'session_rollup'", + "UPDATE entity_sync_state SET payload_hash = '' WHERE entity_kind = 'session_rollup'", [], )?; self.set_metadata_value(SESSION_ROLLUPS_BACKFILL_KEY, "1")?; @@ -90,14 +92,6 @@ impl Store { // and updated_at, so where the hosted row survived, the resend is // a no-op upsert that writes nothing. let existing = self.session_rollup_payload_hashes()?; - for session_id in existing.keys() { - if !grouped.contains_key(session_id) { - self.conn.execute( - "DELETE FROM session_rollups WHERE session_id = ?1", - params![session_id], - )?; - } - } let hashes = grouped .values() .flatten() @@ -105,9 +99,26 @@ impl Store { .collect::>(); let titles = self.session_titles_for_hashes(&hashes)?; let updated_at = Utc::now(); - for (session_id, events) in &grouped { - let rollup = build_session_rollup(events, &titles, updated_at); - let payload_hash = session_rollup_content_hash(&rollup)?; + let rollups = grouped + .values() + .map(|events| build_session_rollup(events, &titles, updated_at)) + .filter(session_rollup_is_identifiable) + .collect::>(); + let kept = rollups + .iter() + .map(|rollup| rollup.session_id.as_str()) + .collect::>(); + for session_id in existing.keys() { + if !kept.contains(session_id.as_str()) { + self.conn.execute( + "DELETE FROM session_rollups WHERE session_id = ?1", + params![session_id], + )?; + } + } + for rollup in &rollups { + let session_id = &rollup.session_id; + let payload_hash = session_rollup_content_hash(rollup)?; if existing.get(session_id) == Some(&payload_hash) { self.conn.execute( "UPDATE session_rollups SET dirty = 1 WHERE session_id = ?1", @@ -115,9 +126,9 @@ impl Store { )?; continue; } - self.write_session_rollup(&rollup, &payload_hash, 1)?; + self.write_session_rollup(rollup, &payload_hash, 1)?; } - Ok(grouped.len() as u64) + Ok(rollups.len() as u64) }) } @@ -151,6 +162,13 @@ impl Store { let titles = self.session_titles_for_hashes(&hashes)?; for (session_id, events) in present { let rollup = build_session_rollup(&events, &titles, Utc::now()); + if !session_rollup_is_identifiable(&rollup) { + self.conn.execute( + "DELETE FROM session_rollups WHERE session_id = ?1", + params![session_id], + )?; + continue; + } let payload_hash = session_rollup_content_hash(&rollup)?; let existing: Option = self .conn @@ -680,6 +698,13 @@ fn resolve_title( (None, None) } +/// A session with neither a project nor a title cannot be told apart from any +/// other, so it is not kept. Cursor cloud agents are the case: the export names +/// the agent by id only and records no project. +fn session_rollup_is_identifiable(rollup: &SessionRollupV1) -> bool { + rollup.project.is_some() || rollup.title.is_some() +} + /// An event whose session identity is its own provider record belongs to no /// session. Cursor's usage export names no session, so each local row is keyed /// by itself, and treating those rows as sessions listed every request as one. diff --git a/crates/statsai-store/src/tests/rollups/sessions.rs b/crates/statsai-store/src/tests/rollups/sessions.rs index 6c399e7..06353de 100644 --- a/crates/statsai-store/src/tests/rollups/sessions.rs +++ b/crates/statsai-store/src/tests/rollups/sessions.rs @@ -4,10 +4,28 @@ use statsai_core::{ TaskSpanId, TASK_SPAN_SCHEMA_VERSION, }; +/// Gives the event a session and a project. Sessions with neither a project +/// nor a title are not kept, so tests of other behavior need one. fn stamp_session(event: &mut UsageEvent, raw_id: &str) { let hash = hash_text(raw_id); event.session.session_id = format!("session_{}", &hash[..24]); event.session.local_session_id_hash = Some(hash); + if event.project.is_none() { + event.project = Some(session_test_project()); + } +} + +fn session_test_project() -> ProjectInfo { + ProjectInfo { + project_id: "project-sessions".to_string(), + project_label: None, + repo_remote_hash: None, + repo_label: None, + branch_hash: None, + branch_label: None, + path_hash: Some("path-sessions".to_string()), + path_label: None, + } } fn message_runtime(user: u64, assistant: u64) -> RuntimeInfo { @@ -629,4 +647,42 @@ fn session_backfill_resends_every_session_to_each_target() { .pending_session_rollups_for_sync("http", "https://api.example.test", &sessions) .expect("pending after backfill"); assert_eq!(pending, sessions); + + // The acknowledgement survives, so a session gone from the next snapshot + // is still retired on the target. + let snapshot = statsai_core::SyncAuthoritativeSnapshot { + session_rollup_ids: Some(Vec::new()), + ..statsai_core::SyncAuthoritativeSnapshot::default() + }; + assert!(store + .sync_target_has_retired_entities("http", "https://api.example.test", &snapshot) + .expect("retired")); +} + +#[test] +fn sessions_without_project_or_title_are_not_kept() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "cursor", + "test", + "0", + Path::new("/tmp/session-unidentifiable"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 11, 9, 0, 0) + .single() + .expect("start"); + let mut agent = test_store_event(&source, start, "agent"); + stamp_session(&mut agent, "cursor_agent:agent-2"); + agent.project = None; + store.insert_event(&agent).expect("agent"); + assert!(store.dirty_session_rollups().expect("sessions").is_empty()); + + agent.session.title = Some("Upgrade the importer".to_string()); + store.insert_event(&agent).expect("titled agent"); + assert_eq!(store.dirty_session_rollups().expect("titled").len(), 1); + + assert_eq!(store.rebuild_session_rollups().expect("rebuild"), 1); } diff --git a/docs/sync-contract.md b/docs/sync-contract.md index 3ff97f8..26f8097 100644 --- a/docs/sync-contract.md +++ b/docs/sync-contract.md @@ -30,7 +30,7 @@ counts, project, bounded title, and `title_source` (`event`, `task_span`, or `archive`). `duration_seconds` is null when a session recorded only a start; the backend must accept null before a collector sends it. Events keyed by their own provider record, such as Cursor's usage-export rows, belong to no session -and are not sent. Prompts, responses, paths, and raw provider session ids stay on +and are not sent, and neither is a session with no project and no title. Prompts, responses, paths, and raw provider session ids stay on the device. Session retirement is not implied by `sync_batch.v6`. A snapshot retires hosted sessions only when it includes `session_rollup_ids`. An empty array prunes them. Omitting the key leaves hosted sessions in place, which is From 34ed37ff55d2dc0b8aa190a8887ad984a5276ee2 Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:19:53 +0300 Subject: [PATCH 09/16] Measure session active time from recorded turns Session duration was the wall-clock span from first to last event, so a thread resumed weeks later read as hundreds of hours. Rollups now also carry active_seconds: the union of the turn intervals the provider recorded, each from the prompt until the agent's last message. Idle time between turns does not count, and nothing guesses where a session went idle. Codex records each turn's start and completion and OpenCode each message's. Claude Code records one timestamp per message, so its events now carry turn_started_at, the prompt that started their turn; tool results and meta records continue a turn. OpenCode's whole-session fallback events carry a span, not a turn, and are left out. statsai sessions reports the median active time and shows active time and span per session, sorts by active time, and formats values past two days in days. --- README.md | 6 +- crates/statsai-adapters/src/claude/parse.rs | 33 +++++- .../src/claude/tests/parse.rs | 47 ++++++++ crates/statsai-adapters/src/event.rs | 1 + crates/statsai-core/src/tests/support.rs | 1 + crates/statsai-core/src/tests/types.rs | 5 + crates/statsai-core/src/types/event.rs | 5 + crates/statsai-core/src/types/session.rs | 6 + crates/statsai-core/src/types/sync.rs | 7 ++ crates/statsai-daemon/src/tests.rs | 1 + .../src/pricing/tests/support.rs | 1 + .../statsai-store/src/quota/tests/support.rs | 1 + crates/statsai-store/src/rollups/sessions.rs | 60 +++++++++- crates/statsai-store/src/tests/events.rs | 1 + .../src/tests/rollups/sessions.rs | 103 ++++++++++++++++++ crates/statsai-store/src/tests/support.rs | 1 + crates/statsai-sync/src/lib.rs | 1 + crates/statsai/src/cli/sessions.rs | 20 ++-- crates/statsai/src/cli/tests/sessions.rs | 10 ++ crates/statsai/src/cli/tests/support.rs | 1 + crates/statsai/src/lib.rs | 1 + .../tests/cli_surface/schema-sync-batch.json | 20 +++- crates/statsai/tests/report_cli.rs | 1 + docs/sync-contract.md | 7 +- 24 files changed, 324 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 1b6bfc4..1670ac0 100644 --- a/README.md +++ b/README.md @@ -165,8 +165,10 @@ with `statsai sync --include-tasks`, the current `sync_batch.v5` payload may include bounded conversation-derived task titles, summary previews, and todo excerpts. Hosted session sync is separate and off by default. `statsai sync --include-sessions` adds `sessions` rows to `sync_batch.v6`: -hashed session ids, token and request totals, duration, message counts, and a -bounded title. Titles come from the provider's own session name (Codex thread +hashed session ids, token and request totals, active time and wall-clock span, +message counts, and a bounded title. Active time is the union of the agent's +turns, each from the prompt until its last message; idle time between turns is +not counted. Titles come from the provider's own session name (Codex thread names, Claude Code and OpenCode session titles) or a local task title. Cursor's usage export names no session, so its local rows are not sessions, and a session with neither a project nor a title is not kept. diff --git a/crates/statsai-adapters/src/claude/parse.rs b/crates/statsai-adapters/src/claude/parse.rs index 6d09111..c0da728 100644 --- a/crates/statsai-adapters/src/claude/parse.rs +++ b/crates/statsai-adapters/src/claude/parse.rs @@ -49,6 +49,7 @@ pub(crate) fn parse_claude_file( // Later snapshots of one provider record replace the event. User lines // counted on the first snapshot have to ride along with that replacement. let mut last_usage_users = HashMap::::new(); + let mut turn_started_at = HashMap::>::new(); loop { let line_status = read_bounded_jsonl_line(&mut reader, &mut line_bytes, MAX_JSONL_RECORD_BYTES)?; @@ -87,6 +88,11 @@ pub(crate) fn parse_claude_file( session_titles.remember(session_raw, rank, title); continue; } + if claude_record_is_prompt(&value) { + if let Some(prompted_at) = timestamp_from_nested_value(&value) { + turn_started_at.insert(session_raw.clone(), prompted_at); + } + } let Some(usage_value) = value .pointer("/message/usage") .or_else(|| value.get("usage")) @@ -157,7 +163,8 @@ pub(crate) fn parse_claude_file( ) }, ); - let event = usage_event( + let prompted_at = turn_started_at.get(&session_raw).copied(); + let mut event = usage_event( ctx.adapter, ctx.source, ctx.options, @@ -181,6 +188,7 @@ pub(crate) fn parse_claude_file( dedupe_salt: None, }, ); + event.session.turn_started_at = prompted_at.filter(|prompted_at| *prompted_at <= timestamp); push_deduped(ctx.scan, ctx.seen, event, duplicate_selection); } @@ -260,6 +268,29 @@ fn claude_session_raw(value: &Value, path: &Path) -> String { .unwrap_or_else(|| fallback_session_id(path)) } +/// A prompt starts a turn: typed text, an image, or a slash command. Tool +/// results arrive as `user` records too, but they continue the turn, as do +/// meta records and compaction summaries. +fn claude_record_is_prompt(value: &Value) -> bool { + if !claude_record_is_user(value) + || value.get("toolUseResult").is_some() + || value.get("isMeta").and_then(Value::as_bool) == Some(true) + || value.get("isCompactSummary").and_then(Value::as_bool) == Some(true) + { + return false; + } + match value.pointer("/message/content") { + Some(Value::String(text)) => !text.trim().is_empty(), + Some(Value::Array(blocks)) => blocks.iter().any(|block| { + matches!( + block.get("type").and_then(Value::as_str), + Some("text" | "image") + ) + }), + _ => false, + } +} + fn claude_record_is_user(value: &Value) -> bool { value.get("type").and_then(Value::as_str) == Some("user") || value.pointer("/message/role").and_then(Value::as_str) == Some("user") diff --git a/crates/statsai-adapters/src/claude/tests/parse.rs b/crates/statsai-adapters/src/claude/tests/parse.rs index f0a8946..a7930cc 100644 --- a/crates/statsai-adapters/src/claude/tests/parse.rs +++ b/crates/statsai-adapters/src/claude/tests/parse.rs @@ -1251,3 +1251,50 @@ fn claude_session_titles_reach_events_read_from_other_transcripts() { Some("Design tool/plugin") ); } + +#[test] +fn claude_events_carry_the_prompt_that_started_their_turn() { + let dir = tempfile::tempdir().expect("tempdir"); + let projects = dir.path().join("projects"); + std::fs::create_dir_all(&projects).expect("projects"); + let mut file = File::create(projects.join("session.jsonl")).expect("session file"); + for line in [ + r#"{"type":"user","timestamp":"2026-05-01T10:00:00Z","sessionId":"turns","message":{"role":"user","content":"Fix the parser"}}"#, + r#"{"type":"assistant","timestamp":"2026-05-01T10:01:00Z","sessionId":"turns","message":{"id":"m1","usage":{"input_tokens":1,"output_tokens":2}}}"#, + r#"{"type":"user","timestamp":"2026-05-01T10:02:00Z","sessionId":"turns","toolUseResult":{},"message":{"role":"user","content":[{"type":"tool_result","content":"ok"}]}}"#, + r#"{"type":"assistant","timestamp":"2026-05-01T10:03:00Z","sessionId":"turns","message":{"id":"m2","usage":{"input_tokens":1,"output_tokens":2}}}"#, + r#"{"type":"user","timestamp":"2026-05-01T15:00:00Z","sessionId":"turns","message":{"role":"user","content":[{"type":"text","text":"Now the tests"}]}}"#, + r#"{"type":"assistant","timestamp":"2026-05-01T15:04:00Z","sessionId":"turns","message":{"id":"m3","usage":{"input_tokens":1,"output_tokens":2}}}"#, + ] { + writeln!(file, "{line}").expect("write line"); + } + + let source = SourceLocation::local_adapter( + CLAUDE_CODE_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let scan = scan_claude_source(&ClaudeCodeAdapter, &source, &options()).expect("scan"); + let mut turns = scan + .events + .iter() + .map(|event| { + ( + event.session.started_at.to_rfc3339(), + event.session.turn_started_at.map(|at| at.to_rfc3339()), + ) + }) + .collect::>(); + turns.sort(); + let prompt = |hour: &str| Some(format!("2026-05-01T{hour}:00:00+00:00")); + assert_eq!( + turns, + vec![ + ("2026-05-01T10:01:00+00:00".to_string(), prompt("10")), + ("2026-05-01T10:03:00+00:00".to_string(), prompt("10")), + ("2026-05-01T15:04:00+00:00".to_string(), prompt("15")), + ] + ); +} diff --git a/crates/statsai-adapters/src/event.rs b/crates/statsai-adapters/src/event.rs index d93052c..c30591d 100644 --- a/crates/statsai-adapters/src/event.rs +++ b/crates/statsai-adapters/src/event.rs @@ -381,6 +381,7 @@ pub(crate) fn usage_event( started_at: session_started_at, ended_at: parts.session_ended_at, duration_seconds: parts.duration_seconds, + turn_started_at: None, }, model: parts.model, runtime: parts.runtime, diff --git a/crates/statsai-core/src/tests/support.rs b/crates/statsai-core/src/tests/support.rs index c183d4e..ce65aee 100644 --- a/crates/statsai-core/src/tests/support.rs +++ b/crates/statsai-core/src/tests/support.rs @@ -43,6 +43,7 @@ pub(super) fn test_event( started_at, ended_at: None, duration_seconds: None, + turn_started_at: None, }, model: None, usage: UsageCounts { diff --git a/crates/statsai-core/src/tests/types.rs b/crates/statsai-core/src/tests/types.rs index c7a1051..e852347 100644 --- a/crates/statsai-core/src/tests/types.rs +++ b/crates/statsai-core/src/tests/types.rs @@ -80,6 +80,7 @@ fn session_rollup_uses_snake_case_contract() { started_at: started, ended_at: started, duration_seconds: Some(0), + active_seconds: None, usage: UsageCounts::default(), requests: 1, cost: CostInfo { @@ -114,6 +115,7 @@ fn session_rollup_uses_snake_case_contract() { "started_at", "ended_at", "duration_seconds", + "active_seconds", "usage", "requests", "cost", @@ -158,6 +160,7 @@ fn session_rollup_unknown_duration_serializes_as_null() { started_at: started, ended_at: started, duration_seconds: None, + active_seconds: None, usage: UsageCounts::default(), requests: 0, cost: CostInfo { @@ -622,6 +625,7 @@ fn sample_session_rollup(started: DateTime) -> SessionRollupV1 { started_at: started, ended_at: started, duration_seconds: Some(0), + active_seconds: None, usage: UsageCounts::default(), requests: 1, cost: CostInfo { @@ -809,6 +813,7 @@ fn sanitized_session_rollup_uses_ingest_allowlist() { "started_at", "ended_at", "duration_seconds", + "active_seconds", "usage", "requests", "cost", diff --git a/crates/statsai-core/src/types/event.rs b/crates/statsai-core/src/types/event.rs index 3f66831..acc7695 100644 --- a/crates/statsai-core/src/types/event.rs +++ b/crates/statsai-core/src/types/event.rs @@ -33,6 +33,11 @@ pub struct SessionInfo { pub started_at: DateTime, pub ended_at: Option>, pub duration_seconds: Option, + /// When the prompt that started this event's turn was sent, for providers + /// that record one timestamp per message. The session's active time runs + /// from here to the event. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub turn_started_at: Option>, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] diff --git a/crates/statsai-core/src/types/session.rs b/crates/statsai-core/src/types/session.rs index 2b5a0a5..e2b8913 100644 --- a/crates/statsai-core/src/types/session.rs +++ b/crates/statsai-core/src/types/session.rs @@ -31,9 +31,15 @@ pub struct SessionRollupV1 { pub provider_account_id: Option, pub started_at: DateTime, pub ended_at: DateTime, + /// Wall-clock span from the first event to the last, idle time included. /// `None` when the session recorded only a start. Unknown durations go out /// as null; ingest stores them without an end so they stay out of medians. pub duration_seconds: Option, + /// Time the agent was working: the union of its turns, each from the prompt + /// until the agent's last message, as the provider recorded them. Idle time + /// between turns does not count. `None` when no turn times were recorded. + #[serde(default)] + pub active_seconds: Option, /// Closed usage object. Cache-lifetime splits stay local; the ingest allowlist /// does not include them. #[serde(with = "session_usage_serde")] diff --git a/crates/statsai-core/src/types/sync.rs b/crates/statsai-core/src/types/sync.rs index 69f91ca..d5bd18a 100644 --- a/crates/statsai-core/src/types/sync.rs +++ b/crates/statsai-core/src/types/sync.rs @@ -311,6 +311,9 @@ fn clamp_session_timestamps(rollup: &mut SessionRollupV1) { rollup.ended_at = rollup.started_at; } rollup.updated_at = clamp_sync_timestamp(rollup.updated_at, latest, fallback); + rollup.active_seconds = rollup + .active_seconds + .map(|value| value.min(SYNC_SAFE_INT_MAX)); // An unknown duration stays unknown; clamped timestamps must not turn it // into a measured zero. if rollup.duration_seconds.is_none() { @@ -323,6 +326,10 @@ fn clamp_session_timestamps(rollup: &mut SessionRollupV1) { rollup.duration_seconds = u64::try_from(seconds) .ok() .map(|value| value.min(SYNC_SAFE_INT_MAX)); + // Active time is a part of the span, so repaired timestamps bound it too. + if let (Some(active), Some(span)) = (rollup.active_seconds, rollup.duration_seconds) { + rollup.active_seconds = Some(active.min(span)); + } } fn sync_timestamp_ok(value: DateTime, latest: DateTime) -> bool { diff --git a/crates/statsai-daemon/src/tests.rs b/crates/statsai-daemon/src/tests.rs index adf4486..040055a 100644 --- a/crates/statsai-daemon/src/tests.rs +++ b/crates/statsai-daemon/src/tests.rs @@ -360,6 +360,7 @@ fn loopback_session(device_id: &str) -> statsai_core::SessionRollupV1 { started_at: now, ended_at: now, duration_seconds: Some(1), + active_seconds: None, usage: UsageCounts::default(), requests: 1, cost: statsai_core::CostInfo { diff --git a/crates/statsai-store/src/pricing/tests/support.rs b/crates/statsai-store/src/pricing/tests/support.rs index 7579b66..828b453 100644 --- a/crates/statsai-store/src/pricing/tests/support.rs +++ b/crates/statsai-store/src/pricing/tests/support.rs @@ -70,6 +70,7 @@ pub(super) fn test_event( started_at, ended_at: None, duration_seconds: None, + turn_started_at: None, }, model: Some(test_model(model)), usage, diff --git a/crates/statsai-store/src/quota/tests/support.rs b/crates/statsai-store/src/quota/tests/support.rs index a435293..c4e634d 100644 --- a/crates/statsai-store/src/quota/tests/support.rs +++ b/crates/statsai-store/src/quota/tests/support.rs @@ -139,6 +139,7 @@ pub(super) fn sample_usage_event( started_at, ended_at: None, duration_seconds: None, + turn_started_at: None, }, model: None, usage: UsageCounts { diff --git a/crates/statsai-store/src/rollups/sessions.rs b/crates/statsai-store/src/rollups/sessions.rs index 08f60cb..2ee6e99 100644 --- a/crates/statsai-store/src/rollups/sessions.rs +++ b/crates/statsai-store/src/rollups/sessions.rs @@ -47,6 +47,7 @@ pub struct SessionStats { pub median_tokens: Option, pub avg_messages: Option, pub median_duration_seconds: Option, + pub median_active_seconds: Option, pub total_messages: u64, pub top_model: Option, } @@ -594,9 +595,17 @@ pub(crate) fn build_session_rollup( entry.1.add(&event.usage, requests, &event.cost); } + // A turn starts at its prompt, which can precede the turn's first event. let started_at = events .iter() - .map(|event| event.session.started_at) + .map(|event| { + event + .session + .turn_started_at + .map_or(event.session.started_at, |turn| { + turn.min(event.session.started_at) + }) + }) .min() .unwrap_or(oldest.session.started_at); let mut ended_at = events @@ -624,6 +633,7 @@ pub(crate) fn build_session_rollup( } else { None }; + let active_seconds = session_active_seconds(events); let provider_account_id = events .iter() .rev() @@ -647,6 +657,7 @@ pub(crate) fn build_session_rollup( started_at, ended_at, duration_seconds, + active_seconds, usage: totals.into_counts(), requests, cost: CostInfo { @@ -678,6 +689,43 @@ pub(crate) fn build_session_rollup( } } +/// Time the agent was working: the union of the intervals the provider +/// recorded for its turns. Codex records a turn's start and completion, +/// OpenCode each message's, and Claude Code events carry the prompt that +/// started their turn. Gaps between turns are idle and do not count; nothing +/// here guesses where a session went idle. +/// +/// An event that aggregates a whole session (OpenCode's `sqlite:session` +/// fallback) carries the session's lifetime, which is a span, not a turn. +fn session_active_seconds(events: &[UsageEvent]) -> Option { + let mut intervals = events + .iter() + .filter(|event| !event.source.source_type.ends_with(":session")) + .filter_map(|event| { + let session = &event.session; + let (start, end) = match session.turn_started_at { + Some(turn) => (turn, session.ended_at.unwrap_or(session.started_at)), + None => (session.started_at, session.ended_at?), + }; + (end >= start).then_some((start, end)) + }) + .collect::>(); + intervals.sort(); + let mut intervals = intervals.into_iter(); + let (mut start, mut end) = intervals.next()?; + let mut total = chrono::Duration::zero(); + for (next_start, next_end) in intervals { + if next_start <= end { + end = end.max(next_end); + } else { + total += end - start; + (start, end) = (next_start, next_end); + } + } + total += end - start; + u64::try_from(total.num_seconds()).ok() +} + fn resolve_title( events: &[UsageEvent], titles: &BTreeMap, @@ -803,10 +851,11 @@ fn sort_sessions(sessions: &mut [SessionRollupV1], sort: SessionSort) { .estimated_micro_usd() .unwrap_or(0) .cmp(&left.cost.estimated_micro_usd().unwrap_or(0)), + // Active time: a thread resumed weeks later is not a long session. SessionSort::Duration => right - .duration_seconds + .active_seconds .unwrap_or(0) - .cmp(&left.duration_seconds.unwrap_or(0)), + .cmp(&left.active_seconds.unwrap_or(0)), SessionSort::Messages => right .total_messages .unwrap_or(0) @@ -844,6 +893,10 @@ fn session_stats(rollups: &[SessionRollupV1]) -> SessionStats { .iter() .filter_map(|rollup| rollup.duration_seconds) .collect::>(); + let mut active = rollups + .iter() + .filter_map(|rollup| rollup.active_seconds) + .collect::>(); let mut model_tokens: BTreeMap<&str, u64> = BTreeMap::new(); for rollup in rollups { if let Some(model) = rollup.primary_model.as_deref() { @@ -864,6 +917,7 @@ fn session_stats(rollups: &[SessionRollupV1]) -> SessionStats { median_tokens, avg_messages, median_duration_seconds: median_u64(&mut durations), + median_active_seconds: median_u64(&mut active), total_messages, top_model, } diff --git a/crates/statsai-store/src/tests/events.rs b/crates/statsai-store/src/tests/events.rs index 8a005d0..eb74fc9 100644 --- a/crates/statsai-store/src/tests/events.rs +++ b/crates/statsai-store/src/tests/events.rs @@ -39,6 +39,7 @@ fn inserts_events_idempotently() { started_at: now, ended_at: None, duration_seconds: None, + turn_started_at: None, }, model: None, usage: UsageCounts { diff --git a/crates/statsai-store/src/tests/rollups/sessions.rs b/crates/statsai-store/src/tests/rollups/sessions.rs index 06353de..9704765 100644 --- a/crates/statsai-store/src/tests/rollups/sessions.rs +++ b/crates/statsai-store/src/tests/rollups/sessions.rs @@ -686,3 +686,106 @@ fn sessions_without_project_or_title_are_not_kept() { assert_eq!(store.rebuild_session_rollups().expect("rebuild"), 1); } + +#[test] +fn active_time_counts_turns_and_skips_idle_gaps() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "claude_code", + "test", + "0", + Path::new("/tmp/session-active"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let at = |hour: u32, minute: u32| { + Utc.with_ymd_and_hms(2026, 6, 12, hour, minute, 0) + .single() + .expect("time") + }; + // Two turns of a Claude-style session, the second resumed hours later: + // each event carries the prompt that started its turn. + for (record, prompt, message) in [ + ("first-a", at(10, 0), at(10, 1)), + ("first-b", at(10, 0), at(10, 5)), + ("second", at(14, 0), at(14, 2)), + ] { + let mut event = test_store_event(&source, message, record); + stamp_session(&mut event, "active-session"); + event.created_at = message; + event.session.turn_started_at = Some(prompt); + store.insert_event(&event).expect("insert"); + } + + let sessions = store.dirty_session_rollups().expect("sessions"); + assert_eq!(sessions.len(), 1); + assert_eq!(sessions[0].started_at, at(10, 0)); + // Wall-clock span keeps the idle afternoon; active time does not. + assert_eq!(sessions[0].duration_seconds, Some(4 * 3600 + 2 * 60)); + assert_eq!(sessions[0].active_seconds, Some(7 * 60)); +} + +#[test] +fn active_time_merges_overlapping_turn_intervals() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-active-overlap"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let at = |minute: u32| { + Utc.with_ymd_and_hms(2026, 6, 13, 9, minute, 0) + .single() + .expect("time") + }; + // Codex-style turns carry their own start and completion. A record + // without an interval adds nothing. + for (record, start, end) in [ + ("turn-a", at(0), Some(at(10))), + ("turn-b", at(5), Some(at(12))), + ("turn-c", at(30), Some(at(31))), + ("record", at(40), None), + ] { + let mut event = test_store_event(&source, start, record); + stamp_session(&mut event, "overlap-session"); + event.created_at = end.unwrap_or(start); + event.session.ended_at = end; + store.insert_event(&event).expect("insert"); + } + + let sessions = store.dirty_session_rollups().expect("sessions"); + assert_eq!(sessions[0].active_seconds, Some(13 * 60)); + assert_eq!(sessions[0].duration_seconds, Some(40 * 60)); +} + +#[test] +fn session_aggregate_events_do_not_count_as_active_time() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "opencode", + "test", + "0", + Path::new("/tmp/session-aggregate"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 14, 9, 0, 0) + .single() + .expect("start"); + let end = start + chrono::Duration::days(3); + let mut aggregate = test_store_event(&source, end, "aggregate"); + stamp_session(&mut aggregate, "aggregate-session"); + aggregate.source.source_type = "sqlite:session".to_string(); + aggregate.session.started_at = start; + aggregate.session.ended_at = Some(end); + aggregate.created_at = end; + store.insert_event(&aggregate).expect("insert"); + + let sessions = store.dirty_session_rollups().expect("sessions"); + assert_eq!(sessions[0].duration_seconds, Some(3 * 86_400)); + assert_eq!(sessions[0].active_seconds, None); +} diff --git a/crates/statsai-store/src/tests/support.rs b/crates/statsai-store/src/tests/support.rs index a3f1e66..0f6fc58 100644 --- a/crates/statsai-store/src/tests/support.rs +++ b/crates/statsai-store/src/tests/support.rs @@ -46,6 +46,7 @@ pub(super) fn test_store_event( started_at: now, ended_at: None, duration_seconds: None, + turn_started_at: None, }, model: None, usage: UsageCounts { diff --git a/crates/statsai-sync/src/lib.rs b/crates/statsai-sync/src/lib.rs index d29f0fc..6fd5a92 100644 --- a/crates/statsai-sync/src/lib.rs +++ b/crates/statsai-sync/src/lib.rs @@ -831,6 +831,7 @@ mod tests { started_at: Utc::now(), ended_at: None, duration_seconds: None, + turn_started_at: None, }, model: None, usage: statsai_core::UsageCounts::default(), diff --git a/crates/statsai/src/cli/sessions.rs b/crates/statsai/src/cli/sessions.rs index 89dc073..5974011 100644 --- a/crates/statsai/src/cli/sessions.rs +++ b/crates/statsai/src/cli/sessions.rs @@ -83,8 +83,8 @@ fn print_sessions_table(view: &SessionsView) { .unwrap_or_else(|| "unknown".to_string()) ); println!( - "median duration: {}", - format_duration(stats.median_duration_seconds) + "median active time: {}", + format_duration(stats.median_active_seconds) ); println!("messages: {}", format_u64(stats.total_messages)); println!( @@ -97,7 +97,7 @@ fn print_sessions_table(view: &SessionsView) { ); println!(); println!( - "{:<16} {:<12} {:<18} {:<24} {:>6} {:>10} {:>10} {:>10} {:>10} {:>10}", + "{:<16} {:<12} {:<18} {:<24} {:>6} {:>10} {:>10} {:>10} {:>10} {:>10} {:>10}", "started", "provider", "model", @@ -107,11 +107,12 @@ fn print_sessions_table(view: &SessionsView) { "output", "total", "est_cost", - "duration" + "active", + "span" ); for session in &view.sessions { println!( - "{:<16} {:<12} {:<18} {:<24} {:>6} {:>10} {:>10} {:>10} {:>10} {:>10}", + "{:<16} {:<12} {:<18} {:<24} {:>6} {:>10} {:>10} {:>10} {:>10} {:>10} {:>10}", session.started_at.format("%Y-%m-%d %H:%M"), truncate_label(&session.provider, 12), truncate_label(session.primary_model.as_deref().unwrap_or("unknown"), 18), @@ -129,19 +130,24 @@ fn print_sessions_table(view: &SessionsView) { .estimated_micro_usd() .map(micro_usd_to_cents_rounded) ), + format_duration(session.active_seconds), format_duration(session.duration_seconds) ); } } -fn format_duration(seconds: Option) -> String { +pub(crate) fn format_duration(seconds: Option) -> String { let Some(seconds) = seconds else { return "unknown".to_string(); }; + let days = seconds / 86_400; let hours = seconds / 3600; let minutes = (seconds % 3600) / 60; let secs = seconds % 60; - if hours > 0 { + // Past two days, hours stop being readable: "20d 18h", not "498h 15m". + if hours >= 48 { + format!("{days}d {:02}h", hours % 24) + } else if hours > 0 { format!("{hours}h {minutes:02}m") } else if minutes > 0 { format!("{minutes}m {secs:02}s") diff --git a/crates/statsai/src/cli/tests/sessions.rs b/crates/statsai/src/cli/tests/sessions.rs index f8d9257..5e4ad9e 100644 --- a/crates/statsai/src/cli/tests/sessions.rs +++ b/crates/statsai/src/cli/tests/sessions.rs @@ -98,3 +98,13 @@ fn sessions_view_reads_rollups_for_the_default_week() { Some(statsai_core::SessionTitleSource::Event) ); } + +#[test] +fn session_durations_switch_to_days_past_two_days() { + use crate::cli::sessions::format_duration; + assert_eq!(format_duration(None), "unknown"); + assert_eq!(format_duration(Some(45)), "45s"); + assert_eq!(format_duration(Some(3 * 60 + 5)), "3m 05s"); + assert_eq!(format_duration(Some(47 * 3600 + 59 * 60)), "47h 59m"); + assert_eq!(format_duration(Some(498 * 3600 + 15 * 60)), "20d 18h"); +} diff --git a/crates/statsai/src/cli/tests/support.rs b/crates/statsai/src/cli/tests/support.rs index 8200dde..b119cdd 100644 --- a/crates/statsai/src/cli/tests/support.rs +++ b/crates/statsai/src/cli/tests/support.rs @@ -231,6 +231,7 @@ pub(super) fn test_event( started_at, ended_at: None, duration_seconds: None, + turn_started_at: None, }, model: None, usage: UsageCounts { diff --git a/crates/statsai/src/lib.rs b/crates/statsai/src/lib.rs index f31a968..bd7852d 100644 --- a/crates/statsai/src/lib.rs +++ b/crates/statsai/src/lib.rs @@ -332,6 +332,7 @@ mod tests { started_at, ended_at: None, duration_seconds: None, + turn_started_at: None, }, model: Some(ModelInfo { name: Some("codex-auto-review".to_string()), diff --git a/crates/statsai/tests/cli_surface/schema-sync-batch.json b/crates/statsai/tests/cli_surface/schema-sync-batch.json index cb9e7b4..2610831 100644 --- a/crates/statsai/tests/cli_surface/schema-sync-batch.json +++ b/crates/statsai/tests/cli_surface/schema-sync-batch.json @@ -1502,6 +1502,14 @@ "string", "null" ] + }, + "turn_started_at": { + "description": "When the prompt that started this event's turn was sent, for providers that record one timestamp per message. The session's active time runs from here to the event.", + "type": [ + "string", + "null" + ], + "format": "date-time" } } }, @@ -1553,6 +1561,16 @@ "usage" ], "properties": { + "active_seconds": { + "description": "Time the agent was working: the union of its turns, each from the prompt until the agent's last message, as the provider recorded them. Idle time between turns does not count. `None` when no turn times were recorded.", + "default": null, + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, "assistant_messages": { "type": [ "integer", @@ -1581,7 +1599,7 @@ "type": "string" }, "duration_seconds": { - "description": "`None` when the session recorded only a start. Unknown durations go out as null; ingest stores them without an end so they stay out of medians.", + "description": "Wall-clock span from the first event to the last, idle time included. `None` when the session recorded only a start. Unknown durations go out as null; ingest stores them without an end so they stay out of medians.", "type": [ "integer", "null" diff --git a/crates/statsai/tests/report_cli.rs b/crates/statsai/tests/report_cli.rs index 6b3ad66..b139515 100644 --- a/crates/statsai/tests/report_cli.rs +++ b/crates/statsai/tests/report_cli.rs @@ -60,6 +60,7 @@ fn test_event( started_at, ended_at: None, duration_seconds: None, + turn_started_at: None, }, model: None, usage: UsageCounts { diff --git a/docs/sync-contract.md b/docs/sync-contract.md index 26f8097..8c19b48 100644 --- a/docs/sync-contract.md +++ b/docs/sync-contract.md @@ -27,8 +27,11 @@ The same batch schema carries opt-in `sessions` rows. Each row is `session_rollup.v1` with a closed key set: hashed session id, provider, source, account, time range, duration, usage, requests, cost, models, message counts, project, bounded title, and `title_source` (`event`, `task_span`, or -`archive`). `duration_seconds` is null when a session recorded only a start; -the backend must accept null before a collector sends it. Events keyed by their +`archive`). `duration_seconds` is the wall-clock span and is null when a +session recorded only a start; the backend must accept null before a collector +sends it. `active_seconds` is the time the agent was working, the union of the +turns the provider recorded, and is null when none were; collectors before it +omit the key. Events keyed by their own provider record, such as Cursor's usage-export rows, belong to no session and are not sent, and neither is a session with no project and no title. Prompts, responses, paths, and raw provider session ids stay on the device. Session retirement is not implied by `sync_batch.v6`. A snapshot From 14e4b528147ea40c8eb42cb696e9217f1d557288 Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:50:56 +0300 Subject: [PATCH 10/16] Stop counting late turn completions as active time Codex writes a turn's completion when the thread next runs, so a turn could read as days long. A reported turn duration now bounds the turn's interval: a turn Codex measured at 21 minutes no longer counts the four days until its completion was written. Turns without a reported duration now end at their last record of work. Your next prompt, which can land inside the open turn just before Codex completes it, starts the next turn and is not counted as work. --- .../statsai-adapters/src/codex/parse/mod.rs | 15 ++++++++- .../statsai-adapters/src/codex/parse/types.rs | 3 ++ .../src/codex/tests/parse/mod.rs | 5 +-- .../src/codex/tests/parse/session.rs | 32 +++++++++++++++++++ crates/statsai-store/src/rollups/sessions.rs | 11 ++++++- .../src/tests/rollups/sessions.rs | 30 +++++++++++++++++ 6 files changed, 92 insertions(+), 4 deletions(-) diff --git a/crates/statsai-adapters/src/codex/parse/mod.rs b/crates/statsai-adapters/src/codex/parse/mod.rs index c068d43..9ec0211 100644 --- a/crates/statsai-adapters/src/codex/parse/mod.rs +++ b/crates/statsai-adapters/src/codex/parse/mod.rs @@ -559,6 +559,7 @@ pub(crate) fn parse_codex_file( accumulated_usage: record.usage.clone(), prompt_previews: Vec::new(), last_activity_at: record.timestamp, + last_work_at: started_at, usage_lines: record .usage .as_ref() @@ -583,6 +584,11 @@ pub(crate) fn parse_codex_file( } turn.timestamp_inferred |= record.timestamp_inferred; turn.last_activity_at = record.timestamp; + // Your next prompt can land inside the open turn just before Codex + // completes it; that is the start of the next turn, not work. + if !record.is_task_complete && record.message_role.as_deref() != Some("user") { + turn.last_work_at = record.timestamp; + } if record.project.is_some() { turn.project = record.project.clone(); } @@ -684,8 +690,15 @@ pub(crate) fn parse_codex_file( consumed_usage_lines.insert(record.line_number); } let explicit_duration_ms = record.task_duration_ms; + // Without a reported duration, the turn ran until its last record + // of work, not until the completion Codex may write on resume. + let worked_until = if turn.last_work_at > turn.started_at { + turn.last_work_at.min(completed_at) + } else { + completed_at + }; let duration_ms = explicit_duration_ms - .or_else(|| codex_duration_from_turn_timestamps(turn.started_at, completed_at)); + .or_else(|| codex_duration_from_turn_timestamps(turn.started_at, worked_until)); let latency_source = explicit_duration_ms .map(|_| LatencySource::Explicit) .or_else(|| duration_ms.map(|_| LatencySource::Inferred)); diff --git a/crates/statsai-adapters/src/codex/parse/types.rs b/crates/statsai-adapters/src/codex/parse/types.rs index 2b7749e..ba0d1aa 100644 --- a/crates/statsai-adapters/src/codex/parse/types.rs +++ b/crates/statsai-adapters/src/codex/parse/types.rs @@ -108,6 +108,9 @@ pub(crate) struct ActiveCodexTurn { pub(crate) accumulated_usage: Option, pub(crate) prompt_previews: Vec, pub(crate) last_activity_at: DateTime, + /// The turn's last record before its completion. Codex writes a turn's + /// completion when the thread next runs, which can be days after the work. + pub(crate) last_work_at: DateTime, pub(crate) usage_lines: Vec, /// `token_count` lines inside the turn that carry no usage, because they /// repeat a total or pair with a `token_usage_record`. Their quota diff --git a/crates/statsai-adapters/src/codex/tests/parse/mod.rs b/crates/statsai-adapters/src/codex/tests/parse/mod.rs index 5f8906a..698d8ae 100644 --- a/crates/statsai-adapters/src/codex/tests/parse/mod.rs +++ b/crates/statsai-adapters/src/codex/tests/parse/mod.rs @@ -705,7 +705,8 @@ fn codex_rollout_turns_match_interleaved_records_by_session_id() { .to_rfc3339(), "2026-05-01T00:00:03+00:00" ); - assert_eq!(events[0].session.duration_seconds, Some(3)); + // Without a reported duration the turn ends at its last record of work. + assert_eq!(events[0].session.duration_seconds, Some(2)); assert_eq!(events[1].usage.total_tokens, Some(280)); assert_eq!( @@ -728,7 +729,7 @@ fn codex_rollout_turns_match_interleaved_records_by_session_id() { .to_rfc3339(), "2026-05-01T00:00:05+00:00" ); - assert_eq!(events[1].session.duration_seconds, Some(4)); + assert_eq!(events[1].session.duration_seconds, Some(3)); } #[test] diff --git a/crates/statsai-adapters/src/codex/tests/parse/session.rs b/crates/statsai-adapters/src/codex/tests/parse/session.rs index ff219f5..9c9445a 100644 --- a/crates/statsai-adapters/src/codex/tests/parse/session.rs +++ b/crates/statsai-adapters/src/codex/tests/parse/session.rs @@ -478,3 +478,35 @@ fn codex_subagents_borrow_their_parent_thread_name() { ); assert_eq!(title_for("child-orphan"), None); } + +#[test] +fn codex_turns_without_a_reported_duration_end_at_their_last_work() { + let dir = tempfile::tempdir().expect("tempdir"); + let codex_root = dir.path().join("codex"); + let sessions = codex_root.join("sessions"); + std::fs::create_dir_all(&sessions).expect("sessions"); + let mut file = File::create(sessions.join("resumed.jsonl")).expect("rollout"); + for line in [ + r#"{"timestamp":"2026-06-01T08:00:00Z","type":"session_meta","payload":{"id":"resumed-thread"}}"#, + r#"{"timestamp":"2026-06-01T08:00:00Z","type":"turn_context","payload":{"model":"gpt-5"}}"#, + r#"{"timestamp":"2026-06-01T08:00:01Z","type":"event_msg","payload":{"type":"task_started","started_at":"2026-06-01T08:00:01Z"}}"#, + r#"{"timestamp":"2026-06-01T08:02:01Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":60,"cached_input_tokens":20,"output_tokens":30,"reasoning_output_tokens":10,"total_tokens":120},"total_token_usage":{"input_tokens":60,"cached_input_tokens":20,"output_tokens":30,"reasoning_output_tokens":10,"total_tokens":120}}}}"#, + // The next prompt and the completion are written three days later, + // when the thread runs again, with no duration. + r#"{"timestamp":"2026-06-04T09:00:00Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"Next step"}]}}"#, + r#"{"timestamp":"2026-06-04T09:00:00Z","type":"event_msg","payload":{"type":"task_complete","completed_at":"2026-06-04T09:00:00Z"}}"#, + ] { + writeln!(file, "{line}").expect("write line"); + } + + let source = SourceLocation::local_adapter( + CODEX_PROVIDER, + "test", + "0", + &codex_root, + LocationOrigin::Configured, + ); + let scan = scan_codex_source(&CodexAdapter, &source, &options_without_tasks()).expect("scan"); + assert_eq!(scan.events.len(), 1); + assert_eq!(scan.events[0].session.duration_seconds, Some(120)); +} diff --git a/crates/statsai-store/src/rollups/sessions.rs b/crates/statsai-store/src/rollups/sessions.rs index 2ee6e99..bba5a60 100644 --- a/crates/statsai-store/src/rollups/sessions.rs +++ b/crates/statsai-store/src/rollups/sessions.rs @@ -703,10 +703,19 @@ fn session_active_seconds(events: &[UsageEvent]) -> Option { .filter(|event| !event.source.source_type.ends_with(":session")) .filter_map(|event| { let session = &event.session; - let (start, end) = match session.turn_started_at { + let (start, mut end) = match session.turn_started_at { Some(turn) => (turn, session.ended_at.unwrap_or(session.started_at)), None => (session.started_at, session.ended_at?), }; + // A reported duration wins over the completion time. Codex writes a + // turn's completion when the thread next runs, so a turn that + // worked 21 minutes can complete four days after it started. + if let Some(reported) = session + .duration_seconds + .and_then(|seconds| i64::try_from(seconds).ok()) + { + end = end.min(start + chrono::Duration::seconds(reported)); + } (end >= start).then_some((start, end)) }) .collect::>(); diff --git a/crates/statsai-store/src/tests/rollups/sessions.rs b/crates/statsai-store/src/tests/rollups/sessions.rs index 9704765..f46de60 100644 --- a/crates/statsai-store/src/tests/rollups/sessions.rs +++ b/crates/statsai-store/src/tests/rollups/sessions.rs @@ -789,3 +789,33 @@ fn session_aggregate_events_do_not_count_as_active_time() { assert_eq!(sessions[0].duration_seconds, Some(3 * 86_400)); assert_eq!(sessions[0].active_seconds, None); } + +#[test] +fn reported_turn_duration_bounds_a_late_completion() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-late-completion"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 7, 23, 17, 16, 0) + .single() + .expect("start"); + // Codex reported 21 minutes of work but wrote the completion when the + // thread resumed four days later. + let completed = start + chrono::Duration::days(4); + let mut turn = test_store_event(&source, start, "late-turn"); + stamp_session(&mut turn, "late-session"); + turn.session.ended_at = Some(completed); + turn.session.duration_seconds = Some(21 * 60); + turn.created_at = completed; + store.insert_event(&turn).expect("insert"); + + let sessions = store.dirty_session_rollups().expect("sessions"); + assert_eq!(sessions[0].active_seconds, Some(21 * 60)); + assert_eq!(sessions[0].duration_seconds, Some(4 * 86_400)); +} From f820c2c8dba59df0923220d2df4090bf268e246c Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Sat, 26 Sep 2026 17:04:11 +0300 Subject: [PATCH 11/16] Fix session stats edge cases and rescan for the new session metadata statsai sessions panicked on an empty store or a filter with no matches: then_some evaluated the average-tokens division even with no sessions. The top model credited each session's whole total to its primary model, so a session split 60/40 between two models counted only one. Stats now sum each session's per-model usage and fall back to the primary model when a session has none. The Codex and Claude parser revisions advance so existing installs re-read unchanged transcripts and pick up titles, message counts, and turn timing on their next scan. --- crates/statsai-adapters/src/cache.rs | 8 ++- crates/statsai-store/src/rollups/sessions.rs | 29 ++++++--- .../src/tests/rollups/sessions.rs | 62 +++++++++++++++++++ 3 files changed, 89 insertions(+), 10 deletions(-) diff --git a/crates/statsai-adapters/src/cache.rs b/crates/statsai-adapters/src/cache.rs index 370c00d..2c286ca 100644 --- a/crates/statsai-adapters/src/cache.rs +++ b/crates/statsai-adapters/src/cache.rs @@ -11,8 +11,12 @@ pub(crate) const SCAN_CACHE_SIGNATURE_VERSION: &str = "scan-cache.v1"; // activity-invocations.v34: Codex usage skips repeated token_count totals and // counts token_usage_record lines, including compaction, instead of the // token_count each one pairs with. -pub(crate) const CODEX_SCAN_CACHE_PARSER_REVISION: &str = "activity-invocations.v34"; -pub(crate) const CLAUDE_SCAN_CACHE_PARSER_REVISION: &str = "activity-invocations.v28"; +// session-metadata.v35: Codex events carry thread names, sub-agents their +// parent's name, and turns without a reported duration end at their last work. +pub(crate) const CODEX_SCAN_CACHE_PARSER_REVISION: &str = "session-metadata.v35"; +// session-metadata.v29: Claude events carry the session's custom or AI title +// and the prompt that started their turn. +pub(crate) const CLAUDE_SCAN_CACHE_PARSER_REVISION: &str = "session-metadata.v29"; // activity-invocations.v20: session rows report their message count as requests and // are priced per message, so long-context tiers stop being decided session-wide. pub(crate) const OPENCODE_SCAN_CACHE_PARSER_REVISION: &str = "activity-invocations.v20"; diff --git a/crates/statsai-store/src/rollups/sessions.rs b/crates/statsai-store/src/rollups/sessions.rs index bba5a60..703214c 100644 --- a/crates/statsai-store/src/rollups/sessions.rs +++ b/crates/statsai-store/src/rollups/sessions.rs @@ -906,23 +906,36 @@ fn session_stats(rollups: &[SessionRollupV1]) -> SessionStats { .iter() .filter_map(|rollup| rollup.active_seconds) .collect::>(); - let mut model_tokens: BTreeMap<&str, u64> = BTreeMap::new(); + // Each session's per-model usage, not its primary model's whole total: a + // session that is 60% one model and 40% another counts both. A session + // without per-model usage counts its primary model. + let mut model_tokens: BTreeMap = BTreeMap::new(); for rollup in rollups { - if let Some(model) = rollup.primary_model.as_deref() { - let tokens = rollup.usage.computed_total(); - let entry = model_tokens.entry(model).or_default(); - *entry = (*entry).saturating_add(tokens); + let mut counted = false; + for usage in &rollup.models { + if let Some(label) = model_label(&usage.model) { + let entry = model_tokens.entry(label).or_default(); + *entry = (*entry).saturating_add(usage.usage.computed_total()); + counted = true; + } + } + if !counted { + if let Some(model) = rollup.primary_model.clone() { + let entry = model_tokens.entry(model).or_default(); + *entry = (*entry).saturating_add(rollup.usage.computed_total()); + } } } let top_model = model_tokens .into_iter() - .max_by(|left, right| left.1.cmp(&right.1).then_with(|| right.0.cmp(left.0))) - .map(|(model, _)| model.to_string()); + .max_by(|left, right| left.1.cmp(&right.1).then_with(|| right.0.cmp(&left.0))) + .map(|(model, _)| model); SessionStats { sessions, total_tokens, total_cost_micro_usd, - avg_tokens: (sessions > 0).then_some(total_tokens / sessions), + // Lazily: `then_some` would divide by zero when nothing matches. + avg_tokens: (sessions > 0).then(|| total_tokens / sessions), median_tokens, avg_messages, median_duration_seconds: median_u64(&mut durations), diff --git a/crates/statsai-store/src/tests/rollups/sessions.rs b/crates/statsai-store/src/tests/rollups/sessions.rs index f46de60..8076dae 100644 --- a/crates/statsai-store/src/tests/rollups/sessions.rs +++ b/crates/statsai-store/src/tests/rollups/sessions.rs @@ -819,3 +819,65 @@ fn reported_turn_duration_bounds_a_late_completion() { assert_eq!(sessions[0].active_seconds, Some(21 * 60)); assert_eq!(sessions[0].duration_seconds, Some(4 * 86_400)); } + +#[test] +fn session_stats_handle_no_matching_sessions() { + let store = Store::in_memory().expect("store"); + let stats = store + .session_stats_in_period(None, Utc::now(), &SessionFilter::default()) + .expect("stats"); + assert_eq!(stats.sessions, 0); + assert_eq!(stats.avg_tokens, None); + assert_eq!(stats.top_model, None); +} + +#[test] +fn top_model_sums_each_sessions_per_model_usage() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-top-model"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 15, 9, 0, 0) + .single() + .expect("start"); + let model = |name: &str| ModelInfo { + name: Some(name.to_string()), + normalized_name: Some(name.to_string()), + provider_model_id: None, + speed: None, + reasoning_level: None, + reasoning_level_raw: None, + }; + // One session is model-a 60 and model-b 40; another is model-b 50. By + // primary model alone, model-a would win 100 to 50. + for (record, session, name, tokens) in [ + ("mixed-a", "mixed-session", "model-a", 60), + ("mixed-b", "mixed-session", "model-b", 40), + ("single-b", "single-session", "model-b", 50), + ] { + let mut event = test_store_event(&source, start, record); + stamp_session(&mut event, session); + event.model = Some(model(name)); + event.usage = UsageCounts { + input_tokens: Some(tokens), + total_tokens: Some(tokens), + ..UsageCounts::default() + }; + store.insert_event(&event).expect("insert"); + } + + let stats = store + .session_stats_in_period( + None, + start + chrono::Duration::days(1), + &SessionFilter::default(), + ) + .expect("stats"); + assert_eq!(stats.top_model.as_deref(), Some("model-b")); +} From cb2ccc184afdb63ff9147347244d4de600c54a04 Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Sat, 26 Sep 2026 17:27:28 +0300 Subject: [PATCH 12/16] Keep a session's project when its latest event has none The rollup took its project from the latest event only. A later event without project metadata erased it, and a session left with neither a project nor a title is not kept, so its usage dropped out of reports and the next snapshot. The project now comes from the latest event that has a stable one. --- crates/statsai-store/src/rollups/sessions.rs | 8 +++++- .../src/tests/rollups/sessions.rs | 28 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/crates/statsai-store/src/rollups/sessions.rs b/crates/statsai-store/src/rollups/sessions.rs index 703214c..2262597 100644 --- a/crates/statsai-store/src/rollups/sessions.rs +++ b/crates/statsai-store/src/rollups/sessions.rs @@ -638,7 +638,13 @@ pub(crate) fn build_session_rollup( .iter() .rev() .find_map(|event| event.provider_account_id.clone()); - let project = newest.project.clone().filter(project_has_stable_identity); + // The latest event that knows its project: a later event without project + // metadata must not erase it, since a session with neither a project nor + // a title is not kept. + let project = events + .iter() + .rev() + .find_map(|event| event.project.clone().filter(project_has_stable_identity)); let (title, title_source) = resolve_title(events, titles); let models = model_buckets .into_iter() diff --git a/crates/statsai-store/src/tests/rollups/sessions.rs b/crates/statsai-store/src/tests/rollups/sessions.rs index 8076dae..5ba2a17 100644 --- a/crates/statsai-store/src/tests/rollups/sessions.rs +++ b/crates/statsai-store/src/tests/rollups/sessions.rs @@ -881,3 +881,31 @@ fn top_model_sums_each_sessions_per_model_usage() { .expect("stats"); assert_eq!(stats.top_model.as_deref(), Some("model-b")); } + +#[test] +fn a_later_event_without_a_project_keeps_the_sessions_project() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-project-gap"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 16, 9, 0, 0) + .single() + .expect("start"); + let mut first = test_store_event(&source, start, "with-project"); + stamp_session(&mut first, "project-gap-session"); + store.insert_event(&first).expect("first"); + let mut latest = test_store_event(&source, start + chrono::Duration::minutes(5), "no-project"); + stamp_session(&mut latest, "project-gap-session"); + latest.project = None; + store.insert_event(&latest).expect("latest"); + + let sessions = store.dirty_session_rollups().expect("sessions"); + assert_eq!(sessions.len(), 1); + assert_eq!(sessions[0].project, Some(session_test_project())); +} From 1abb3d5e9c0bfdf4414549e185a723c411a63ea0 Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Sat, 26 Sep 2026 17:43:51 +0300 Subject: [PATCH 13/16] Rebuild shared sessions when a source is deleted, and rescan OpenCode Deleting a source removed session rollups by the source of their newest event, but a session can hold events from more than one source. Deleting the older source left its tokens in the rollup, and deleting the newer one dropped the session despite surviving events. The sessions a deletion touches are now rebuilt from the events that remain. The OpenCode parser revision advances so existing installs re-read unchanged databases and pick up per-session message counts. --- crates/statsai-adapters/src/cache.rs | 7 +- crates/statsai-store/src/events.rs | 16 ++++- crates/statsai-store/src/rollups/sessions.rs | 14 ---- .../src/tests/rollups/sessions.rs | 64 +++++++++++++++++++ 4 files changed, 83 insertions(+), 18 deletions(-) diff --git a/crates/statsai-adapters/src/cache.rs b/crates/statsai-adapters/src/cache.rs index 2c286ca..f7adbf1 100644 --- a/crates/statsai-adapters/src/cache.rs +++ b/crates/statsai-adapters/src/cache.rs @@ -17,9 +17,10 @@ pub(crate) const CODEX_SCAN_CACHE_PARSER_REVISION: &str = "session-metadata.v35" // session-metadata.v29: Claude events carry the session's custom or AI title // and the prompt that started their turn. pub(crate) const CLAUDE_SCAN_CACHE_PARSER_REVISION: &str = "session-metadata.v29"; -// activity-invocations.v20: session rows report their message count as requests and -// are priced per message, so long-context tiers stop being decided session-wide. -pub(crate) const OPENCODE_SCAN_CACHE_PARSER_REVISION: &str = "activity-invocations.v20"; +// session-metadata.v21: events carry each session's user and assistant message +// counts. (v20: session rows report their message count as requests and are +// priced per message, so long-context tiers stop being decided session-wide.) +pub(crate) const OPENCODE_SCAN_CACHE_PARSER_REVISION: &str = "session-metadata.v21"; // Revisit Grok sessions whose Fast request was left unpriced because modelsUsed // also retained a selected standard model that made no inference. pub(crate) const GROK_BUILD_SCAN_CACHE_PARSER_REVISION: &str = "grok-fast-pricing.v23"; diff --git a/crates/statsai-store/src/events.rs b/crates/statsai-store/src/events.rs index e31c566..92483f8 100644 --- a/crates/statsai-store/src/events.rs +++ b/crates/statsai-store/src/events.rs @@ -609,6 +609,20 @@ impl Store { ) -> Result { self.with_immediate_transaction(|| { let mut impact = EventDeletionImpact::default(); + // A session can hold events from more than one source, so the + // sessions these events belong to are rebuilt from what remains + // rather than deleted by the source their newest event came from. + let mut session_ids = BTreeSet::new(); + for source_id in source_ids { + let mut statement = self.conn.prepare(&format!( + "SELECT DISTINCT {EVENT_SESSION_ID_SQL} FROM usage_events WHERE source_id = ?1" + ))?; + let rows = statement + .query_map(params![&source_id.0], |row| row.get::<_, Option>(0))?; + for row in rows { + session_ids.extend(row?); + } + } for source_id in source_ids { // `usage_events_source_idx` covers this, so naming the ids costs an // index range read rather than a pass over the payloads. @@ -629,7 +643,7 @@ impl Store { )? as u64; } self.delete_sync_rollups_for_sources_in_tx(source_ids)?; - self.delete_session_rollups_for_sources_in_tx(source_ids)?; + self.refresh_session_rollups_for_keys(&session_ids)?; Ok(impact) }) } diff --git a/crates/statsai-store/src/rollups/sessions.rs b/crates/statsai-store/src/rollups/sessions.rs index 2262597..df108c6 100644 --- a/crates/statsai-store/src/rollups/sessions.rs +++ b/crates/statsai-store/src/rollups/sessions.rs @@ -297,20 +297,6 @@ impl Store { Ok(updated) } - pub(crate) fn delete_session_rollups_for_sources_in_tx( - &self, - source_ids: &[statsai_core::SourceId], - ) -> Result { - let mut deleted = 0u64; - for source_id in source_ids { - deleted += self.conn.execute( - "DELETE FROM session_rollups WHERE source_id = ?1", - params![&source_id.0], - )? as u64; - } - Ok(deleted) - } - pub fn session_rollups_in_period( &self, since: Option>, diff --git a/crates/statsai-store/src/tests/rollups/sessions.rs b/crates/statsai-store/src/tests/rollups/sessions.rs index 5ba2a17..e2f51ae 100644 --- a/crates/statsai-store/src/tests/rollups/sessions.rs +++ b/crates/statsai-store/src/tests/rollups/sessions.rs @@ -909,3 +909,67 @@ fn a_later_event_without_a_project_keeps_the_sessions_project() { assert_eq!(sessions.len(), 1); assert_eq!(sessions[0].project, Some(session_test_project())); } + +#[test] +fn deleting_a_source_rebuilds_sessions_it_shared() { + let store = Store::in_memory().expect("store"); + let older = SourceLocation::local_adapter( + "claude_code", + "test", + "0", + Path::new("/tmp/session-shared-older"), + LocationOrigin::Configured, + ); + let newer = SourceLocation::local_adapter( + "claude_code", + "test", + "0", + Path::new("/tmp/session-shared-newer"), + LocationOrigin::Configured, + ); + store.upsert_source(&older).expect("older source"); + store.upsert_source(&newer).expect("newer source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 17, 9, 0, 0) + .single() + .expect("start"); + for (source, record, at, tokens) in [ + (&older, "older-event", start, 10), + ( + &newer, + "newer-event", + start + chrono::Duration::minutes(5), + 7, + ), + ] { + let mut event = test_store_event(source, at, record); + stamp_session(&mut event, "shared-session"); + event.usage.total_tokens = Some(tokens); + store.insert_event(&event).expect("insert"); + } + let total = |store: &Store| { + store + .all_session_rollups() + .expect("sessions") + .iter() + .map(|session| session.usage.computed_total()) + .collect::>() + }; + assert_eq!(total(&store), vec![17]); + + // The older source's tokens leave the session, which the newer one keeps. + store + .delete_events_for_sources(std::slice::from_ref(&older.source_id)) + .expect("delete older"); + assert_eq!(total(&store), vec![7]); + + // And the other way round: the surviving events keep the session. + let mut again = test_store_event(&older, start, "older-again"); + stamp_session(&mut again, "shared-session"); + again.usage.total_tokens = Some(10); + store.insert_event(&again).expect("reinsert older"); + store + .delete_events_for_sources(std::slice::from_ref(&newer.source_id)) + .expect("delete newer"); + assert_eq!(total(&store), vec![10]); +} From fefc0dd7f9fee3d3bfb5ff189242ddf9c5db0573 Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Sat, 26 Sep 2026 18:26:10 +0300 Subject: [PATCH 14/16] Track provider session names apart from usage, and refresh on span removal Session names live in files the scan cache does not tie to usage. Codex keeps them in session_index.jsonl, which changes no rollout, and Claude Code writes a rename into the resumed transcript while the usage stays in an unchanged one. Either rename left the old title in place until a full scan. Scans now report session names apart from events: Codex from the index on every scan, Claude from each transcript it reads. The store keeps them in a session_names table (schema 27), writes only changed names, refreshes the sessions they name, and ranks them first when titling a session. Index names skip the prompt cleaning task titles use, which dropped a name like "Design tool/plugin". Deleting task spans now refreshes the sessions they titled, so a title from a removed source no longer stays cached and eligible for sync. --- crates/statsai-adapters/src/claude/mod.rs | 3 +- crates/statsai-adapters/src/claude/parse.rs | 19 +++- .../src/claude/tests/parse.rs | 31 +++++++ crates/statsai-adapters/src/codex/mod.rs | 15 ++- .../src/codex/parse/session.rs | 29 ++++-- .../src/codex/tests/parse/session.rs | 29 ++++++ crates/statsai-adapters/src/lib.rs | 2 + crates/statsai-core/src/types/session.rs | 12 +++ crates/statsai-daemon/src/watch/scan.rs | 1 + crates/statsai-store/src/lib.rs | 1 + crates/statsai-store/src/migrations/mod.rs | 3 +- crates/statsai-store/src/migrations/v2.rs | 16 ++++ .../src/pricing/tests/repricing.rs | 2 +- crates/statsai-store/src/rollups/sessions.rs | 74 +++++++++++++-- crates/statsai-store/src/scan_state.rs | 1 + crates/statsai-store/src/tasks/spans.rs | 19 ++++ .../src/tests/rollups/sessions.rs | 92 +++++++++++++++++++ crates/statsai-store/src/tests/scan_state.rs | 2 + crates/statsai/src/cli/scan/mod.rs | 1 + 19 files changed, 329 insertions(+), 23 deletions(-) diff --git a/crates/statsai-adapters/src/claude/mod.rs b/crates/statsai-adapters/src/claude/mod.rs index 98f8f5c..b7c20e8 100644 --- a/crates/statsai-adapters/src/claude/mod.rs +++ b/crates/statsai-adapters/src/claude/mod.rs @@ -230,7 +230,8 @@ pub(crate) fn scan_claude_source( )?; } } - session_titles.apply(&mut scan.events); + let session_names = session_titles.apply(&mut scan.events); + scan.session_names.extend(session_names); if let Some(candidate) = claude_stats_cache_candidate(&root, &cache_namespaces) { if options.should_scan(&candidate.cache_key) { diff --git a/crates/statsai-adapters/src/claude/parse.rs b/crates/statsai-adapters/src/claude/parse.rs index c0da728..d0b44cf 100644 --- a/crates/statsai-adapters/src/claude/parse.rs +++ b/crates/statsai-adapters/src/claude/parse.rs @@ -224,16 +224,18 @@ impl ClaudeSessionTitles { } } - pub(crate) fn apply(self, events: &mut [statsai_core::UsageEvent]) { - if self.0.is_empty() { - return; - } + /// Stamps the titles on this scan's events and returns them as session + /// names, which also reach sessions whose events this scan did not read. + pub(crate) fn apply( + self, + events: &mut [statsai_core::UsageEvent], + ) -> Vec { let titles_by_hash = self .0 .into_iter() .map(|(session_raw, (_, title))| (hash_text(&session_raw), title)) .collect::>(); - for event in events { + for event in events.iter_mut() { if let Some(title) = event .session .local_session_id_hash @@ -243,6 +245,13 @@ impl ClaudeSessionTitles { event.session.title = Some(title.clone()); } } + titles_by_hash + .into_iter() + .map(|(local_session_id_hash, title)| statsai_core::SessionName { + local_session_id_hash, + title, + }) + .collect() } } diff --git a/crates/statsai-adapters/src/claude/tests/parse.rs b/crates/statsai-adapters/src/claude/tests/parse.rs index a7930cc..2ee6170 100644 --- a/crates/statsai-adapters/src/claude/tests/parse.rs +++ b/crates/statsai-adapters/src/claude/tests/parse.rs @@ -1298,3 +1298,34 @@ fn claude_events_carry_the_prompt_that_started_their_turn() { ] ); } + +#[test] +fn claude_title_only_transcripts_report_session_names() { + let dir = tempfile::tempdir().expect("tempdir"); + let projects = dir.path().join("projects"); + std::fs::create_dir_all(&projects).expect("projects"); + // A resumed transcript holding only the rename; the usage lives elsewhere. + let mut file = File::create(projects.join("resumed.jsonl")).expect("resumed"); + writeln!( + file, + r#"{{"type":"custom-title","customTitle":"Renamed session","sessionId":"renamed"}}"# + ) + .expect("write title"); + + let source = SourceLocation::local_adapter( + CLAUDE_CODE_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let scan = scan_claude_source(&ClaudeCodeAdapter, &source, &options()).expect("scan"); + assert!(scan.events.is_empty()); + assert_eq!( + scan.session_names, + vec![statsai_core::SessionName { + local_session_id_hash: statsai_core::hash_text("renamed"), + title: "Renamed session".to_string(), + }] + ); +} diff --git a/crates/statsai-adapters/src/codex/mod.rs b/crates/statsai-adapters/src/codex/mod.rs index e6a2655..e4fa4f8 100644 --- a/crates/statsai-adapters/src/codex/mod.rs +++ b/crates/statsai-adapters/src/codex/mod.rs @@ -124,7 +124,20 @@ pub(crate) fn scan_codex_source( let root = codex_source_root(&source_path); let cache_namespaces = scan_cache_namespaces(source, adapter.version()); // Thread names title sessions as well as tasks, so they load either way. - let thread_titles = load_codex_thread_titles(&root); + // The index is re-read on every scan and its names go out as session + // names: a rename there changes no rollout, so the scan cache would keep + // every rollout's old title. + let thread_names = load_codex_thread_names(&root); + let thread_titles = codex_thread_titles_from_names(&thread_names); + scan.session_names + .extend( + thread_names + .into_iter() + .map(|(session_id, title)| statsai_core::SessionName { + local_session_id_hash: hash_text(&session_id), + title, + }), + ); let mut indexed_candidates = Vec::new(); for (index, candidate) in codex_jsonl_candidates(source, &source_path, &cache_namespaces)? .into_iter() diff --git a/crates/statsai-adapters/src/codex/parse/session.rs b/crates/statsai-adapters/src/codex/parse/session.rs index 1a47ff0..1b51bd9 100644 --- a/crates/statsai-adapters/src/codex/parse/session.rs +++ b/crates/statsai-adapters/src/codex/parse/session.rs @@ -1,12 +1,14 @@ use super::*; -pub(crate) fn load_codex_thread_titles(root: &Path) -> HashMap { +/// Thread names from `session_index.jsonl`, as the provider shows them. Task +/// titles clean them further with the prompt rules; session names do not. +pub(crate) fn load_codex_thread_names(root: &Path) -> HashMap { let index_path = root.join("session_index.jsonl"); let Ok(file) = File::open(&index_path) else { return HashMap::new(); }; let mut reader = BufReader::new(file); - let mut titles = HashMap::new(); + let mut names = HashMap::new(); let mut line_bytes = Vec::new(); while let Ok(line_status) = read_bounded_jsonl_line(&mut reader, &mut line_bytes, MAX_JSONL_RECORD_BYTES) @@ -26,14 +28,25 @@ pub(crate) fn load_codex_thread_titles(root: &Path) -> HashMap { let Some(session_id) = value.get("id").and_then(Value::as_str) else { continue; }; - let Some(title) = value.get("thread_name").and_then(Value::as_str) else { - continue; - }; - if let Some(title) = summarize_task_text(Some(title), 90) { - titles.insert(session_id.to_string(), title); + if let Some(name) = statsai_core::provider_session_name( + value.get("thread_name").and_then(Value::as_str), + 90, + ) { + names.insert(session_id.to_string(), name); } } - titles + names +} + +pub(crate) fn codex_thread_titles_from_names( + names: &HashMap, +) -> HashMap { + names + .iter() + .filter_map(|(session_id, name)| { + summarize_task_text(Some(name), 90).map(|title| (session_id.clone(), title)) + }) + .collect() } /// Codex never names its sub-agents. A spawned agent records its parent diff --git a/crates/statsai-adapters/src/codex/tests/parse/session.rs b/crates/statsai-adapters/src/codex/tests/parse/session.rs index 9c9445a..79006b2 100644 --- a/crates/statsai-adapters/src/codex/tests/parse/session.rs +++ b/crates/statsai-adapters/src/codex/tests/parse/session.rs @@ -510,3 +510,32 @@ fn codex_turns_without_a_reported_duration_end_at_their_last_work() { assert_eq!(scan.events.len(), 1); assert_eq!(scan.events[0].session.duration_seconds, Some(120)); } + +#[test] +fn codex_session_index_names_are_reported_on_every_scan() { + let dir = tempfile::tempdir().expect("tempdir"); + let codex_root = dir.path().join("codex"); + std::fs::create_dir_all(codex_root.join("sessions")).expect("sessions"); + std::fs::write( + codex_root.join("session_index.jsonl"), + "{\"id\":\"thread-1\",\"thread_name\":\"Design tool/plugin\"}\n", + ) + .expect("session index"); + + let source = SourceLocation::local_adapter( + CODEX_PROVIDER, + "test", + "0", + &codex_root, + LocationOrigin::Configured, + ); + let scan = scan_codex_source(&CodexAdapter, &source, &options_without_tasks()).expect("scan"); + // No rollout changed, yet the index name is reported, uncleaned. + assert_eq!( + scan.session_names, + vec![statsai_core::SessionName { + local_session_id_hash: statsai_core::hash_text("thread-1"), + title: "Design tool/plugin".to_string(), + }] + ); +} diff --git a/crates/statsai-adapters/src/lib.rs b/crates/statsai-adapters/src/lib.rs index d5ca92a..01d4962 100644 --- a/crates/statsai-adapters/src/lib.rs +++ b/crates/statsai-adapters/src/lib.rs @@ -168,6 +168,8 @@ pub struct AdapterScan { pub events: Vec, pub summaries: Vec, pub task_spans: Vec, + /// Session names read apart from usage events; see [`SessionName`]. + pub session_names: Vec, pub quota_observations: Vec, pub activity_invocations: Vec, pub activity_coverage: Vec, diff --git a/crates/statsai-core/src/types/session.rs b/crates/statsai-core/src/types/session.rs index e2b8913..830335a 100644 --- a/crates/statsai-core/src/types/session.rs +++ b/crates/statsai-core/src/types/session.rs @@ -17,6 +17,18 @@ pub enum SessionTitleSource { Archive, } +/// A name the provider holds for a session, read apart from its usage events. +/// +/// Names live in files the scan cache does not tie to usage: Codex keeps them +/// in `session_index.jsonl`, and Claude Code writes a renamed title into the +/// transcript that was resumed, while the usage stays in an unchanged one. A +/// rename therefore reaches the session without re-parsing its events. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SessionName { + pub local_session_id_hash: String, + pub title: String, +} + /// One local session, aggregated from the usage events that share its hashed id. /// /// `session_id` is the hashed `session_…` identity already stored on each event. diff --git a/crates/statsai-daemon/src/watch/scan.rs b/crates/statsai-daemon/src/watch/scan.rs index d144a74..d0aa4b4 100644 --- a/crates/statsai-daemon/src/watch/scan.rs +++ b/crates/statsai-daemon/src/watch/scan.rs @@ -491,6 +491,7 @@ fn rescan_changed_sources_with_adapters_and_commit_store_and_dependencies( reconciled_file_hashes: &reconciled_file_hashes, events: &scan.events, summaries: &scan.summaries, + session_names: &scan.session_names, activity_invocations: &scan.activity_invocations, activity_coverage: &scan.activity_coverage, activity_persist_mode: ActivityPersistMode::for_scan( diff --git a/crates/statsai-store/src/lib.rs b/crates/statsai-store/src/lib.rs index 6c6ca78..8817b9b 100644 --- a/crates/statsai-store/src/lib.rs +++ b/crates/statsai-store/src/lib.rs @@ -290,6 +290,7 @@ pub struct ScanFileReplacement<'a> { pub reconciled_file_hashes: &'a [String], pub events: &'a [UsageEvent], pub summaries: &'a [UsageSummary], + pub session_names: &'a [statsai_core::SessionName], pub activity_invocations: &'a [ActivityInvocationV1], pub activity_coverage: &'a [ActivityCoverageV1], pub activity_persist_mode: ActivityPersistMode, diff --git a/crates/statsai-store/src/migrations/mod.rs b/crates/statsai-store/src/migrations/mod.rs index 3c31083..34264d8 100644 --- a/crates/statsai-store/src/migrations/mod.rs +++ b/crates/statsai-store/src/migrations/mod.rs @@ -8,7 +8,7 @@ mod v2; pub(crate) use v1::*; pub(crate) use v2::*; -pub const CURRENT_SCHEMA_VERSION: i64 = 26; +pub const CURRENT_SCHEMA_VERSION: i64 = 27; pub fn migrate(conn: &Connection) -> Result<()> { if let Some(current) = existing_schema_version(conn)? { @@ -151,6 +151,7 @@ fn apply_migration(conn: &Connection, version: i64) -> Result<()> { 24 => apply_migration_024(conn), 25 => apply_migration_025(conn), 26 => apply_migration_026(conn), + 27 => apply_migration_027(conn), _ => bail!("unsupported schema migration version {version}"), } } diff --git a/crates/statsai-store/src/migrations/v2.rs b/crates/statsai-store/src/migrations/v2.rs index c41d923..a4c8b46 100644 --- a/crates/statsai-store/src/migrations/v2.rs +++ b/crates/statsai-store/src/migrations/v2.rs @@ -578,3 +578,19 @@ pub(crate) fn apply_migration_026(conn: &Connection) -> Result<()> { ))?; Ok(()) } + +/// Provider-held session names, keyed by the hashed session id. Session +/// rollups read them ahead of titles stamped on events, so a rename reaches the +/// session without re-parsing its transcripts. +pub(crate) fn apply_migration_027(conn: &Connection) -> Result<()> { + conn.execute_batch( + r#" + CREATE TABLE IF NOT EXISTS session_names ( + local_session_id_hash TEXT PRIMARY KEY, + title TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + "#, + )?; + Ok(()) +} diff --git a/crates/statsai-store/src/pricing/tests/repricing.rs b/crates/statsai-store/src/pricing/tests/repricing.rs index a4c8094..c6131ca 100644 --- a/crates/statsai-store/src/pricing/tests/repricing.rs +++ b/crates/statsai-store/src/pricing/tests/repricing.rs @@ -384,7 +384,7 @@ fn daily_rollups_are_unused_by_report_and_sync_paths() { assert_eq!(events[0].cost, expected_review_cost(started_at)); // Pin: bump this when CURRENT_SCHEMA_VERSION changes, after confirming // the daily_rollups table is still unused by report/sync/snapshot. - assert_eq!(CURRENT_SCHEMA_VERSION, 26); + assert_eq!(CURRENT_SCHEMA_VERSION, 27); } #[test] diff --git a/crates/statsai-store/src/rollups/sessions.rs b/crates/statsai-store/src/rollups/sessions.rs index df108c6..431ff04 100644 --- a/crates/statsai-store/src/rollups/sessions.rs +++ b/crates/statsai-store/src/rollups/sessions.rs @@ -7,8 +7,9 @@ use chrono::{DateTime, Utc}; use rusqlite::{params, OptionalExtension}; use statsai_core::{ hash_text, project_has_stable_identity, provider_session_title_is_unusable, - task_title_is_generic, Confidence, CostAccumulator, CostInfo, ModelInfo, SessionRollupV1, - SessionTitleSource, SummaryModelUsage, UsageCounts, UsageEvent, SESSION_ROLLUP_SCHEMA_VERSION, + task_title_is_generic, Confidence, CostAccumulator, CostInfo, ModelInfo, SessionName, + SessionRollupV1, SessionTitleSource, SummaryModelUsage, UsageCounts, UsageEvent, + SESSION_ROLLUP_SCHEMA_VERSION, }; use std::collections::{BTreeMap, BTreeSet}; @@ -187,6 +188,36 @@ impl Store { Ok(()) } + /// Records the names providers hold for sessions and refreshes the sessions + /// whose name changed. Unchanged names write nothing. + pub fn upsert_session_names(&self, names: &[SessionName]) -> Result { + if names.is_empty() { + return Ok(0); + } + let mut changed_hashes = Vec::new(); + { + let mut statement = self.conn.prepare( + r#" + INSERT INTO session_names (local_session_id_hash, title, updated_at) + VALUES (?1, ?2, ?3) + ON CONFLICT(local_session_id_hash) DO UPDATE SET + title = excluded.title, + updated_at = excluded.updated_at + WHERE session_names.title IS NOT excluded.title + "#, + )?; + let now = Utc::now().to_rfc3339(); + for name in names { + if statement.execute(params![&name.local_session_id_hash, &name.title, &now])? > 0 { + changed_hashes.push(name.local_session_id_hash.clone()); + } + } + } + let session_ids = self.session_ids_for_local_hashes(&changed_hashes)?; + self.refresh_session_rollups_for_keys(&session_ids)?; + Ok(changed_hashes.len() as u64) + } + pub(crate) fn refresh_session_rollups_for_raw_ids(&self, raw_ids: &[String]) -> Result<()> { let hashes = raw_ids .iter() @@ -484,6 +515,24 @@ impl Store { ); } } + drop(spans); + + // A provider-held name outranks task and archive titles for its session. + let mut names = self + .conn + .prepare("SELECT local_session_id_hash, title FROM session_names")?; + let name_rows = names.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })?; + for row in name_rows { + let (hash, title) = row?; + if !wanted.contains(hash.as_str()) { + continue; + } + if let Some(title) = acceptable_session_title(Some(&title), SessionTitleSource::Event) { + titles.insert(hash, (title, SessionTitleSource::Event)); + } + } Ok(titles) } @@ -727,19 +776,32 @@ fn session_active_seconds(events: &[UsageEvent]) -> Option { u64::try_from(total.num_seconds()).ok() } +/// Precedence: the name the provider holds now, then a title stamped on an +/// event when it was parsed, then a task-span title, then an archive title. fn resolve_title( events: &[UsageEvent], titles: &BTreeMap, ) -> (Option, Option) { + let hashes = || { + events + .iter() + .rev() + .filter_map(|event| event.session.local_session_id_hash.as_deref()) + }; + if let Some(title) = hashes().find_map(|hash| { + titles + .get(hash) + .filter(|(_, source)| *source == SessionTitleSource::Event) + .map(|(title, _)| title.clone()) + }) { + return (Some(title), Some(SessionTitleSource::Event)); + } if let Some(title) = events.iter().rev().find_map(|event| { acceptable_session_title(event.session.title.as_deref(), SessionTitleSource::Event) }) { return (Some(title), Some(SessionTitleSource::Event)); } - for event in events.iter().rev() { - let Some(hash) = event.session.local_session_id_hash.as_deref() else { - continue; - }; + for hash in hashes() { if let Some((title, source)) = titles.get(hash) { return (Some(title.clone()), Some(*source)); } diff --git a/crates/statsai-store/src/scan_state.rs b/crates/statsai-store/src/scan_state.rs index eaa22cb..7118b05 100644 --- a/crates/statsai-store/src/scan_state.rs +++ b/crates/statsai-store/src/scan_state.rs @@ -255,6 +255,7 @@ impl Store { )?; let inserted_events = self.insert_events(replacement.events)?; let written_summaries = self.upsert_summaries(replacement.summaries)?; + self.upsert_session_names(replacement.session_names)?; // A quota observation can point at an event this just deleted, and the // daemon -- the only caller -- never touches quota rows itself, so nothing // else repairs the link. Run after the insert: a record that survives the diff --git a/crates/statsai-store/src/tasks/spans.rs b/crates/statsai-store/src/tasks/spans.rs index 090ee7b..ebcea8f 100644 --- a/crates/statsai-store/src/tasks/spans.rs +++ b/crates/statsai-store/src/tasks/spans.rs @@ -228,13 +228,32 @@ impl Store { let mut delete_spans = self .conn .prepare("DELETE FROM task_spans WHERE span_id = ?1")?; + // A deleted span can have been the source of a session's title, while + // the session's usage survives under another source. + let mut session_of_span = self.conn.prepare( + "SELECT json_extract(payload, '$.session_id') FROM task_spans WHERE span_id = ?1", + )?; + let mut raw_session_ids = Vec::new(); let mut deleted = 0u64; let mut affected_project_buckets = BTreeSet::new(); for target in targets { affected_project_buckets.insert(target.project_bucket.clone()); + if let Some(session_id) = session_of_span + .query_row(params![&target.span_id.0], |row| { + row.get::<_, Option>(0) + }) + .optional()? + .flatten() + { + raw_session_ids.push(session_id); + } delete_links.execute(params![&target.span_id.0])?; deleted += delete_spans.execute(params![&target.span_id.0])? as u64; } + drop(session_of_span); + drop(delete_links); + drop(delete_spans); + self.refresh_session_rollups_for_raw_ids(&raw_session_ids)?; Ok(TaskDeletionImpact { deleted, affected_project_buckets, diff --git a/crates/statsai-store/src/tests/rollups/sessions.rs b/crates/statsai-store/src/tests/rollups/sessions.rs index e2f51ae..22aa5eb 100644 --- a/crates/statsai-store/src/tests/rollups/sessions.rs +++ b/crates/statsai-store/src/tests/rollups/sessions.rs @@ -973,3 +973,95 @@ fn deleting_a_source_rebuilds_sessions_it_shared() { .expect("delete newer"); assert_eq!(total(&store), vec![10]); } + +#[test] +fn deleting_the_span_that_titled_a_session_clears_the_title() { + let store = Store::in_memory().expect("store"); + let usage_source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-span-usage"), + LocationOrigin::Configured, + ); + let span_source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-span-titles"), + LocationOrigin::Configured, + ); + store.upsert_source(&usage_source).expect("usage source"); + store.upsert_source(&span_source).expect("span source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 18, 9, 0, 0) + .single() + .expect("start"); + let raw_id = "span-titled-session"; + let mut event = test_store_event(&usage_source, start, "span-usage"); + stamp_session(&mut event, raw_id); + store.insert_event(&event).expect("insert"); + let title = "Repair the span title cleanup"; + store + .upsert_task_spans(&[task_span(&span_source, raw_id, title, start)]) + .expect("span"); + assert_eq!( + store.all_session_rollups().expect("titled")[0] + .title + .as_deref(), + Some(title) + ); + + let targets = store + .task_span_targets_for_sources(std::slice::from_ref(&span_source.source_id)) + .expect("targets"); + store + .delete_task_span_targets_in_tx(&targets) + .expect("delete spans"); + let sessions = store.all_session_rollups().expect("after"); + assert_eq!(sessions.len(), 1); + assert_eq!(sessions[0].title, None); +} + +#[test] +fn a_provider_name_renames_a_stored_session_without_its_events() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "claude_code", + "test", + "0", + Path::new("/tmp/session-names"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 19, 9, 0, 0) + .single() + .expect("start"); + let mut event = test_store_event(&source, start, "named"); + stamp_session(&mut event, "named-session"); + event.session.title = Some("Title when parsed".to_string()); + store.insert_event(&event).expect("insert"); + + let rename = |title: &str| statsai_core::SessionName { + local_session_id_hash: hash_text("named-session"), + title: title.to_string(), + }; + assert_eq!( + store + .upsert_session_names(&[rename("Renamed later")]) + .expect("rename"), + 1 + ); + let sessions = store.all_session_rollups().expect("sessions"); + assert_eq!(sessions[0].title.as_deref(), Some("Renamed later")); + assert_eq!(sessions[0].title_source, Some(SessionTitleSource::Event)); + + // An unchanged name writes nothing. + assert_eq!( + store + .upsert_session_names(&[rename("Renamed later")]) + .expect("same"), + 0 + ); +} diff --git a/crates/statsai-store/src/tests/scan_state.rs b/crates/statsai-store/src/tests/scan_state.rs index 2a8c8ab..179e3fe 100644 --- a/crates/statsai-store/src/tests/scan_state.rs +++ b/crates/statsai-store/src/tests/scan_state.rs @@ -60,6 +60,7 @@ fn scan_file_replacement_rolls_back_deletions_when_cache_update_fails() { reconciled_file_hashes: &[file_hash], events: &[], summaries: &[], + session_names: &[], activity_invocations: &[], activity_coverage: &[], activity_persist_mode: ActivityPersistMode::ReplaceFiles, @@ -325,6 +326,7 @@ fn scan_file_replacement_clears_links_to_events_it_retired() { reconciled_file_hashes: std::slice::from_ref(&file_hash), events: std::slice::from_ref(&surviving), summaries: &[], + session_names: &[], activity_invocations: &[], activity_coverage: &[], activity_persist_mode: ActivityPersistMode::ReplaceFiles, diff --git a/crates/statsai/src/cli/scan/mod.rs b/crates/statsai/src/cli/scan/mod.rs index f4f8a22..303f8d2 100644 --- a/crates/statsai/src/cli/scan/mod.rs +++ b/crates/statsai/src/cli/scan/mod.rs @@ -466,6 +466,7 @@ pub(crate) fn scan_with_adapters( } let insert_started_at = Instant::now(); let insert_result = store.insert_events_with_resolution(&scan.events)?; + store.upsert_session_names(&scan.session_names)?; inserted_count += insert_result.inserted; insert_events_duration_ms += insert_started_at.elapsed().as_millis() as u64; rewrite_quota_usage_event_ids( From b173edf00e73a0b411e862574cc1bebd81d9b919 Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Sat, 26 Sep 2026 19:28:02 +0300 Subject: [PATCH 15/16] Rescan Codex names when only the session index changes A thread rename rewrites session_index.jsonl and no rollout, so a cached scan skipped the adapter and kept the old title. The index is now a scan-cache entry of its own: a change to it selects no rollout but reads names again. It stays out of the conversation archive. --- crates/statsai-adapters/src/codex/mod.rs | 22 +++- .../src/codex/parse/session.rs | 4 +- .../src/codex/tests/parse/mod.rs | 59 ++++++++++ crates/statsai/src/cli/tests/scan/files.rs | 111 ++++++++++++++++++ 4 files changed, 194 insertions(+), 2 deletions(-) diff --git a/crates/statsai-adapters/src/codex/mod.rs b/crates/statsai-adapters/src/codex/mod.rs index e4fa4f8..db0ada1 100644 --- a/crates/statsai-adapters/src/codex/mod.rs +++ b/crates/statsai-adapters/src/codex/mod.rs @@ -63,6 +63,18 @@ impl ProviderAdapter for CodexAdapter { codex_scan_candidates(source, self.version()) } + fn archive_scan_candidates(&self, source: &SourceLocation) -> Result> { + let Some(path_label) = source + .path_label + .as_deref() + .filter(|label| !label.is_empty()) + else { + return Ok(Vec::new()); + }; + let cache_namespaces = scan_cache_namespaces(source, self.version()); + codex_jsonl_candidates(source, Path::new(path_label), &cache_namespaces) + } + fn probe_verified_source_state( &self, source: &SourceLocation, @@ -280,7 +292,15 @@ pub(crate) fn codex_scan_candidates( }; let source_path = PathBuf::from(path_label); let cache_namespaces = scan_cache_namespaces(source, adapter_version); - codex_jsonl_candidates(source, &source_path, &cache_namespaces) + let mut candidates = codex_jsonl_candidates(source, &source_path, &cache_namespaces)?; + // Renaming a thread rewrites only the index, so it is tracked as a file of + // its own: a change to it rescans the source for names without selecting + // any rollout. + let index_path = codex_source_root(&source_path).join(CODEX_SESSION_INDEX_FILE); + if index_path.is_file() { + candidates.push(scan_candidate(index_path, None, &cache_namespaces)); + } + Ok(candidates) } pub(crate) fn codex_jsonl_candidates( diff --git a/crates/statsai-adapters/src/codex/parse/session.rs b/crates/statsai-adapters/src/codex/parse/session.rs index 1b51bd9..49a6c0c 100644 --- a/crates/statsai-adapters/src/codex/parse/session.rs +++ b/crates/statsai-adapters/src/codex/parse/session.rs @@ -1,9 +1,11 @@ use super::*; +pub(crate) const CODEX_SESSION_INDEX_FILE: &str = "session_index.jsonl"; + /// Thread names from `session_index.jsonl`, as the provider shows them. Task /// titles clean them further with the prompt rules; session names do not. pub(crate) fn load_codex_thread_names(root: &Path) -> HashMap { - let index_path = root.join("session_index.jsonl"); + let index_path = root.join(CODEX_SESSION_INDEX_FILE); let Ok(file) = File::open(&index_path) else { return HashMap::new(); }; diff --git a/crates/statsai-adapters/src/codex/tests/parse/mod.rs b/crates/statsai-adapters/src/codex/tests/parse/mod.rs index 698d8ae..f7f42f0 100644 --- a/crates/statsai-adapters/src/codex/tests/parse/mod.rs +++ b/crates/statsai-adapters/src/codex/tests/parse/mod.rs @@ -178,6 +178,65 @@ fn codex_scan_candidates_ignore_auth_json_changes() { assert_eq!(first[0].cache_signature, second[0].cache_signature); } +#[test] +fn codex_session_index_is_a_scan_candidate_of_its_own() { + let dir = tempfile::tempdir().expect("tempdir"); + let sessions = dir.path().join("sessions"); + std::fs::create_dir_all(&sessions).expect("sessions"); + let session_path = sessions.join("session.jsonl"); + std::fs::write( + &session_path, + "{\"timestamp\":\"2026-05-01T00:00:00Z\",\"usage\":{\"input_tokens\":1,\"output_tokens\":2}}\n", + ) + .expect("session"); + let index_path = dir.path().join("session_index.jsonl"); + std::fs::write( + &index_path, + "{\"id\":\"thread-1\",\"thread_name\":\"One\"}\n", + ) + .expect("index one"); + + let source = SourceLocation::local_adapter( + CODEX_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let signature_of = |candidates: &[ScanCandidateFile], path: &Path| { + candidates + .iter() + .find(|candidate| candidate.cache_key == canonical_display(path)) + .map(|candidate| candidate.cache_signature.clone()) + }; + + let first = codex_scan_candidates(&source, "test-adapter").expect("first candidates"); + std::thread::sleep(std::time::Duration::from_millis(5)); + std::fs::write( + &index_path, + "{\"id\":\"thread-1\",\"thread_name\":\"One\"}\n{\"id\":\"thread-1\",\"thread_name\":\"Two\"}\n", + ) + .expect("index two"); + let second = codex_scan_candidates(&source, "test-adapter").expect("second candidates"); + + assert_eq!(first.len(), 2); + // A rename marks the index changed and leaves every rollout cached. + assert_ne!( + signature_of(&first, &index_path), + signature_of(&second, &index_path) + ); + assert_eq!( + signature_of(&first, &session_path), + signature_of(&second, &session_path) + ); + // The index is not a conversation to archive. + let archive = CodexAdapter + .archive_scan_candidates(&source) + .expect("archive candidates"); + assert_eq!(archive.len(), 1); + assert_eq!(archive[0].cache_key, canonical_display(&session_path)); +} + #[test] fn codex_scan_candidates_accept_legacy_auth_dependent_signatures() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/crates/statsai/src/cli/tests/scan/files.rs b/crates/statsai/src/cli/tests/scan/files.rs index 0ab5310..f07be94 100644 --- a/crates/statsai/src/cli/tests/scan/files.rs +++ b/crates/statsai/src/cli/tests/scan/files.rs @@ -772,3 +772,114 @@ fn scan_rewrites_task_span_links_to_canonical_event_ids() { vec![existing_event.event_id.clone()] ); } + +/// The real Codex adapter, pointed at one source instead of the machine's. +struct PinnedCodexAdapter { + source: SourceLocation, +} + +impl ProviderAdapter for PinnedCodexAdapter { + fn id(&self) -> &'static str { + statsai_adapters::CodexAdapter.id() + } + + fn version(&self) -> &'static str { + statsai_adapters::CodexAdapter.version() + } + + fn provider(&self) -> &'static str { + statsai_adapters::CodexAdapter.provider() + } + + fn discover(&self) -> Vec { + vec![self.source.clone()] + } + + fn scan_candidates(&self, source: &SourceLocation) -> Result> { + statsai_adapters::CodexAdapter.scan_candidates(source) + } + + fn scan( + &self, + source: &SourceLocation, + options: &ScanOptions, + ) -> Result { + statsai_adapters::CodexAdapter.scan(source, options) + } +} + +#[test] +fn cached_scan_picks_up_a_codex_thread_rename() { + let dir = tempfile::tempdir().expect("tempdir"); + let codex_root = dir.path().join("codex"); + let sessions = codex_root.join("sessions"); + std::fs::create_dir_all(&sessions).expect("sessions"); + let index_path = codex_root.join("session_index.jsonl"); + std::fs::write( + &index_path, + "{\"id\":\"thread-1\",\"thread_name\":\"First name\"}\n", + ) + .expect("session index"); + std::fs::write( + sessions.join("rollout.jsonl"), + [ + r#"{"timestamp":"2026-06-01T08:00:00Z","type":"session_meta","payload":{"id":"thread-1"}}"#, + r#"{"timestamp":"2026-06-01T08:00:00Z","type":"turn_context","payload":{"model":"gpt-5"}}"#, + r#"{"timestamp":"2026-06-01T08:00:03Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":60,"cached_input_tokens":20,"output_tokens":30,"reasoning_output_tokens":10,"total_tokens":120},"total_token_usage":{"input_tokens":60,"cached_input_tokens":20,"output_tokens":30,"reasoning_output_tokens":10,"total_tokens":120}}}}"#, + ] + .join("\n") + + "\n", + ) + .expect("rollout"); + + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "codex-local-jsonl", + statsai_adapters::CodexAdapter.version(), + &codex_root, + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let scan = || { + scan_with_adapters( + ScanCommand { + provider: None, + include_tasks: false, + preview: false, + no_cache: false, + replace: false, + verbose: false, + explain: false, + }, + &store, + "device-test", + vec![Box::new(PinnedCodexAdapter { + source: source.clone(), + })], + ) + .expect("scan"); + }; + let titles = || { + store + .all_session_rollups() + .expect("sessions") + .into_iter() + .map(|session| session.title) + .collect::>() + }; + + scan(); + assert_eq!(titles(), vec![Some("First name".to_string())]); + + // Codex appends the rename to the index and leaves the rollout alone. + std::thread::sleep(std::time::Duration::from_millis(5)); + std::fs::write( + &index_path, + "{\"id\":\"thread-1\",\"thread_name\":\"First name\"}\n\ + {\"id\":\"thread-1\",\"thread_name\":\"Renamed thread\"}\n", + ) + .expect("renamed index"); + scan(); + assert_eq!(titles(), vec![Some("Renamed thread".to_string())]); +} From 9b582b27ae9c6a6bddade43dc867d50385f387be Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Sat, 26 Sep 2026 20:06:10 +0300 Subject: [PATCH 16/16] Fix session message counts, sub-agent names, title lookups, and project exclusion Claude user-message counts now include only typed prompts. Tool results, meta lines, and compaction summaries had counted as user messages, so a prompt followed by ten tool calls read as eleven. Codex sub-agents record the parent they borrow a name from, and the store joins the parent's current name when the session is built. A parent rename now reaches its sub-agents without re-reading their rollouts, and the parent's name is used as the provider shows it. A session's own name is never replaced by a borrowed one. Task spans and archive conversations store hashed session keys (schema 28), so a session refresh looks up titles for its own sessions instead of reading both tables in full on every write. Excluding projects now retires this device's hosted sessions, which carry project names and paths. Turning off only session sync still leaves them in place. --- crates/statsai-adapters/src/cache.rs | 11 +- crates/statsai-adapters/src/claude/parse.rs | 5 +- .../src/claude/tests/parse.rs | 44 ++++ crates/statsai-adapters/src/codex/mod.rs | 1 + .../statsai-adapters/src/codex/parse/mod.rs | 9 +- .../src/codex/parse/session.rs | 14 +- .../src/codex/tests/parse/session.rs | 25 +- crates/statsai-adapters/src/event.rs | 1 + crates/statsai-core/src/types/session.rs | 6 + crates/statsai-store/src/archive/write.rs | 5 +- crates/statsai-store/src/lib.rs | 2 +- crates/statsai-store/src/migrations/mod.rs | 77 +++++- crates/statsai-store/src/migrations/v2.rs | 61 +++++ .../src/pricing/tests/repricing.rs | 2 +- crates/statsai-store/src/rollups/mod.rs | 1 + crates/statsai-store/src/rollups/sessions.rs | 232 +++++++++++------- .../statsai-store/src/sync_state/pending.rs | 49 +++- crates/statsai-store/src/tasks/spans.rs | 13 +- .../src/tests/rollups/sessions.rs | 57 +++++ crates/statsai/src/cli/args/sync.rs | 2 +- crates/statsai/src/cli/sync/batch.rs | 18 +- crates/statsai/src/cli/sync/mod.rs | 6 +- crates/statsai/src/cli/tests/sync/batch.rs | 67 +++++ crates/statsai/tests/cli_surface/help.txt | 2 +- docs/sync-contract.md | 4 +- 25 files changed, 575 insertions(+), 139 deletions(-) diff --git a/crates/statsai-adapters/src/cache.rs b/crates/statsai-adapters/src/cache.rs index f7adbf1..e3b9edd 100644 --- a/crates/statsai-adapters/src/cache.rs +++ b/crates/statsai-adapters/src/cache.rs @@ -13,10 +13,13 @@ pub(crate) const SCAN_CACHE_SIGNATURE_VERSION: &str = "scan-cache.v1"; // token_count each one pairs with. // session-metadata.v35: Codex events carry thread names, sub-agents their // parent's name, and turns without a reported duration end at their last work. -pub(crate) const CODEX_SCAN_CACHE_PARSER_REVISION: &str = "session-metadata.v35"; -// session-metadata.v29: Claude events carry the session's custom or AI title -// and the prompt that started their turn. -pub(crate) const CLAUDE_SCAN_CACHE_PARSER_REVISION: &str = "session-metadata.v29"; +// session-metadata.v36: sub-agents report the parent they borrow a name from, +// so the name is joined when the session is built. +pub(crate) const CODEX_SCAN_CACHE_PARSER_REVISION: &str = "session-metadata.v36"; +// session-metadata.v30: Claude user-message counts include only typed prompts, +// not tool results, meta lines, or compaction summaries. (v29: events carry the +// session's custom or AI title and the prompt that started their turn.) +pub(crate) const CLAUDE_SCAN_CACHE_PARSER_REVISION: &str = "session-metadata.v30"; // session-metadata.v21: events carry each session's user and assistant message // counts. (v20: session rows report their message count as requests and are // priced per message, so long-context tiers stop being decided session-wide.) diff --git a/crates/statsai-adapters/src/claude/parse.rs b/crates/statsai-adapters/src/claude/parse.rs index d0b44cf..d27275a 100644 --- a/crates/statsai-adapters/src/claude/parse.rs +++ b/crates/statsai-adapters/src/claude/parse.rs @@ -97,7 +97,7 @@ pub(crate) fn parse_claude_file( .pointer("/message/usage") .or_else(|| value.get("usage")) else { - if claude_record_is_user(&value) { + if claude_record_is_prompt(&value) { *pending_user_messages.entry(session_raw).or_default() += 1; } continue; @@ -105,7 +105,7 @@ pub(crate) fn parse_claude_file( ctx.scan.diagnostics.candidate_usage_rows += 1; let usage = claude_usage_counts_from_value(usage_value); if usage.computed_total() == 0 { - if claude_record_is_user(&value) { + if claude_record_is_prompt(&value) { *pending_user_messages.entry(session_raw).or_default() += 1; } ctx.scan.diagnostics.skipped_zero_events += 1; @@ -250,6 +250,7 @@ impl ClaudeSessionTitles { .map(|(local_session_id_hash, title)| statsai_core::SessionName { local_session_id_hash, title, + parent_local_session_id_hash: None, }) .collect() } diff --git a/crates/statsai-adapters/src/claude/tests/parse.rs b/crates/statsai-adapters/src/claude/tests/parse.rs index 2ee6170..9cf184e 100644 --- a/crates/statsai-adapters/src/claude/tests/parse.rs +++ b/crates/statsai-adapters/src/claude/tests/parse.rs @@ -549,6 +549,49 @@ fn claude_user_counts_stay_on_the_kept_streaming_snapshot() { assert_eq!(second.assistant_messages, Some(1)); } +#[test] +fn claude_user_counts_skip_tool_results_meta_lines_and_compaction_summaries() { + let dir = tempfile::tempdir().expect("tempdir"); + let projects = dir.path().join("projects"); + std::fs::create_dir_all(&projects).expect("projects"); + let mut file = File::create(projects.join("session.jsonl")).expect("session"); + for line in [ + r#"{"type":"user","timestamp":"2026-08-05T14:51:00.000Z","sessionId":"session-1","isMeta":true,"message":{"role":"user","content":"Caveat"}}"#, + r#"{"type":"user","timestamp":"2026-08-05T14:51:01.000Z","sessionId":"session-1","isCompactSummary":true,"message":{"role":"user","content":"This session is being continued"}}"#, + r#"{"type":"user","timestamp":"2026-08-05T14:51:02.000Z","sessionId":"session-1","message":{"role":"user","content":"fix the parser"}}"#, + r#"{"timestamp":"2026-08-05T14:51:03.000Z","sessionId":"session-1","uuid":"record-1","requestId":"request-1","message":{"id":"message-1","role":"assistant","model":"claude-opus-5","usage":{"input_tokens":2,"output_tokens":10}}}"#, + r#"{"type":"user","timestamp":"2026-08-05T14:51:04.000Z","sessionId":"session-1","toolUseResult":{"stdout":"ok"},"message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"tool-1","content":"ok"}]}}"#, + r#"{"type":"user","timestamp":"2026-08-05T14:51:05.000Z","sessionId":"session-1","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"tool-2","content":"ok"}]}}"#, + r#"{"timestamp":"2026-08-05T14:51:06.000Z","sessionId":"session-1","uuid":"record-2","requestId":"request-2","message":{"id":"message-2","role":"assistant","model":"claude-opus-5","usage":{"input_tokens":3,"output_tokens":5}}}"#, + ] { + writeln!(file, "{line}").expect("line"); + } + + let source = SourceLocation::local_adapter( + CLAUDE_CODE_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let scan = scan_claude_source(&ClaudeCodeAdapter, &source, &options()).expect("scan"); + + // One typed prompt, followed by two tool results that are not prompts. + let user_messages = scan + .events + .iter() + .map(|event| { + event + .runtime + .as_ref() + .and_then(|runtime| runtime.user_messages) + .unwrap_or(0) + }) + .sum::(); + assert_eq!(scan.events.len(), 2); + assert_eq!(user_messages, 1); +} + #[test] fn claude_streaming_snapshots_with_equal_timestamps_resolve_by_source_line() { let dir = tempfile::tempdir().expect("tempdir"); @@ -1326,6 +1369,7 @@ fn claude_title_only_transcripts_report_session_names() { vec![statsai_core::SessionName { local_session_id_hash: statsai_core::hash_text("renamed"), title: "Renamed session".to_string(), + parent_local_session_id_hash: None, }] ); } diff --git a/crates/statsai-adapters/src/codex/mod.rs b/crates/statsai-adapters/src/codex/mod.rs index db0ada1..3b5befa 100644 --- a/crates/statsai-adapters/src/codex/mod.rs +++ b/crates/statsai-adapters/src/codex/mod.rs @@ -148,6 +148,7 @@ pub(crate) fn scan_codex_source( .map(|(session_id, title)| statsai_core::SessionName { local_session_id_hash: hash_text(&session_id), title, + parent_local_session_id_hash: None, }), ); let mut indexed_candidates = Vec::new(); diff --git a/crates/statsai-adapters/src/codex/parse/mod.rs b/crates/statsai-adapters/src/codex/parse/mod.rs index 9ec0211..054d0fa 100644 --- a/crates/statsai-adapters/src/codex/parse/mod.rs +++ b/crates/statsai-adapters/src/codex/parse/mod.rs @@ -41,7 +41,6 @@ pub(crate) fn parse_codex_file( let mut current_title: Option = None; // Events only: a sub-agent borrows its parent thread's name, but task // titles keep their own prompt-based rules. - let mut subagent_title: Option = None; let mut current_thread_id: Option = None; // Seeded from the file path, replaced by the session's own id as soon as // the `session_meta` line declares one. The embedded id is the same UUID @@ -352,7 +351,9 @@ pub(crate) fn parse_codex_file( .and_then(Value::as_str) .map(ToOwned::to_owned) .or_else(|| thread_titles.get(&session_id).cloned()); - subagent_title = codex_subagent_session_title(&value, &session_id, thread_titles); + ctx.scan + .session_names + .extend(codex_subagent_session_name(&value, &session_id)); current_project = codex_project_context_from_value(&value, &mut project_cache); continue; } @@ -743,7 +744,7 @@ pub(crate) fn parse_codex_file( dedupe_salt: None, }, ); - event.session.title = turn.title.clone().or_else(|| subagent_title.clone()); + event.session.title = turn.title.clone(); let mut linked_quota_lines = turn.usage_lines.clone(); linked_quota_lines.extend_from_slice(&turn.quota_lines); if record.usage.is_some() { @@ -940,7 +941,7 @@ pub(crate) fn parse_codex_file( if consumed_usage_lines.contains(&record.line_number) { continue; } - let session_title = record.session_title.or_else(|| subagent_title.clone()); + let session_title = record.session_title; let mut event = usage_event( ctx.adapter, ctx.source, diff --git a/crates/statsai-adapters/src/codex/parse/session.rs b/crates/statsai-adapters/src/codex/parse/session.rs index 49a6c0c..d5f1d74 100644 --- a/crates/statsai-adapters/src/codex/parse/session.rs +++ b/crates/statsai-adapters/src/codex/parse/session.rs @@ -54,11 +54,12 @@ pub(crate) fn codex_thread_titles_from_names( /// Codex never names its sub-agents. A spawned agent records its parent /// thread, and a guardian review carries the parent's id as `session_id`, so /// both borrow the parent's name, marked with the agent's nickname or kind. -pub(crate) fn codex_subagent_session_title( +/// The name is joined when the session is built, so a parent rename reaches +/// its sub-agents without re-reading their rollouts. +pub(crate) fn codex_subagent_session_name( value: &Value, session_id: &str, - thread_titles: &HashMap, -) -> Option { +) -> Option { let payload = value.get("payload")?; let subagent = payload.pointer("/source/subagent")?; let (parent, label) = if let Some(spawn) = subagent.get("thread_spawn") { @@ -81,8 +82,11 @@ pub(crate) fn codex_subagent_session_title( .unwrap_or("sub-agent"); (parent, label) }; - let parent_title = thread_titles.get(parent)?; - statsai_core::provider_session_name(Some(&format!("{parent_title} · {label}")), 90) + Some(statsai_core::SessionName { + local_session_id_hash: hash_text(session_id), + title: statsai_core::provider_session_name(Some(label), 40)?, + parent_local_session_id_hash: Some(hash_text(parent)), + }) } pub(crate) fn codex_project_context_from_value( diff --git a/crates/statsai-adapters/src/codex/tests/parse/session.rs b/crates/statsai-adapters/src/codex/tests/parse/session.rs index 79006b2..3850d0a 100644 --- a/crates/statsai-adapters/src/codex/tests/parse/session.rs +++ b/crates/statsai-adapters/src/codex/tests/parse/session.rs @@ -458,25 +458,25 @@ fn codex_subagents_borrow_their_parent_thread_name() { LocationOrigin::Configured, ); let scan = scan_codex_source(&CodexAdapter, &source, &options_without_tasks()).expect("scan"); - let title_for = |raw: &str| { + // The parent's name is joined when the session is built, so a rename of + // the parent reaches its sub-agents; here only the link is reported. + let link_for = |raw: &str| { let hash = statsai_core::hash_text(raw); - scan.events + scan.session_names .iter() - .find(|event| event.session.local_session_id_hash.as_deref() == Some(hash.as_str())) - .expect("event") - .session - .title - .clone() + .find(|name| name.local_session_id_hash == hash) + .cloned() }; + let parent = Some(statsai_core::hash_text("parent-1")); assert_eq!( - title_for("child-spawned").as_deref(), - Some("Fix parser bug · Popper") + link_for("child-spawned").map(|name| (name.title, name.parent_local_session_id_hash)), + Some(("Popper".to_string(), parent.clone())) ); assert_eq!( - title_for("child-guardian").as_deref(), - Some("Fix parser bug · guardian") + link_for("child-guardian").map(|name| (name.title, name.parent_local_session_id_hash)), + Some(("guardian".to_string(), parent)) ); - assert_eq!(title_for("child-orphan"), None); + assert_eq!(link_for("child-orphan"), None); } #[test] @@ -536,6 +536,7 @@ fn codex_session_index_names_are_reported_on_every_scan() { vec![statsai_core::SessionName { local_session_id_hash: statsai_core::hash_text("thread-1"), title: "Design tool/plugin".to_string(), + parent_local_session_id_hash: None, }] ); } diff --git a/crates/statsai-adapters/src/event.rs b/crates/statsai-adapters/src/event.rs index c30591d..f38a812 100644 --- a/crates/statsai-adapters/src/event.rs +++ b/crates/statsai-adapters/src/event.rs @@ -134,6 +134,7 @@ pub(crate) fn merge_adapter_scan( push_deduped(target, index, event, DuplicateSelection::KeepFirst); } target.summaries.append(&mut source.summaries); + target.session_names.append(&mut source.session_names); target.task_spans.append(&mut source.task_spans); target .quota_observations diff --git a/crates/statsai-core/src/types/session.rs b/crates/statsai-core/src/types/session.rs index 830335a..40a2be1 100644 --- a/crates/statsai-core/src/types/session.rs +++ b/crates/statsai-core/src/types/session.rs @@ -23,10 +23,16 @@ pub enum SessionTitleSource { /// in `session_index.jsonl`, and Claude Code writes a renamed title into the /// transcript that was resumed, while the usage stays in an unchanged one. A /// rename therefore reaches the session without re-parsing its events. +/// +/// A sub-agent the provider never names borrows its parent's name instead: +/// `parent_local_session_id_hash` points at the parent, `title` is the +/// sub-agent's own label, and the session reads "{parent name} · {label}" +/// with whatever name the parent holds at the time. #[derive(Debug, Clone, PartialEq, Eq)] pub struct SessionName { pub local_session_id_hash: String, pub title: String, + pub parent_local_session_id_hash: Option, } /// One local session, aggregated from the usage events that share its hashed id. diff --git a/crates/statsai-store/src/archive/write.rs b/crates/statsai-store/src/archive/write.rs index e16659c..99543e0 100644 --- a/crates/statsai-store/src/archive/write.rs +++ b/crates/statsai-store/src/archive/write.rs @@ -34,8 +34,8 @@ impl Store { INSERT INTO archive_conversations (conversation_id, provider, source_id, native_conversation_id, title, project_json, started_at, updated_at, completeness, - missing_content_count, imported_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) + missing_content_count, imported_at, session_hash) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12) ON CONFLICT(conversation_id) DO UPDATE SET source_id = excluded.source_id, title = COALESCE(excluded.title, archive_conversations.title), @@ -68,6 +68,7 @@ impl Store { conversation.completeness.as_str(), conversation.missing_content_count, &imported_at, + statsai_core::hash_text(&conversation.native_conversation_id), ], )?; result.conversations += 1; diff --git a/crates/statsai-store/src/lib.rs b/crates/statsai-store/src/lib.rs index 8817b9b..3bc5eb7 100644 --- a/crates/statsai-store/src/lib.rs +++ b/crates/statsai-store/src/lib.rs @@ -26,7 +26,7 @@ pub(crate) use rollups::{ collect_pending_summary_days, event_with_valid_project, is_daily_rollup_summary, is_http_rollup_passthrough_summary, sanitize_summary_for_http_sync, summary_period_bounds, summary_sync_payload_hash, sync_rollup_bucket_key, sync_rollup_project_key, - SyncRollupBucketKey, + task_span_session_hashes, SyncRollupBucketKey, }; pub(crate) use sql::{ begin_immediate_transaction_with_retry, commit_transaction, enable_sqlite_defensive, diff --git a/crates/statsai-store/src/migrations/mod.rs b/crates/statsai-store/src/migrations/mod.rs index 34264d8..6d6b9a1 100644 --- a/crates/statsai-store/src/migrations/mod.rs +++ b/crates/statsai-store/src/migrations/mod.rs @@ -8,7 +8,7 @@ mod v2; pub(crate) use v1::*; pub(crate) use v2::*; -pub const CURRENT_SCHEMA_VERSION: i64 = 27; +pub const CURRENT_SCHEMA_VERSION: i64 = 28; pub fn migrate(conn: &Connection) -> Result<()> { if let Some(current) = existing_schema_version(conn)? { @@ -152,6 +152,7 @@ fn apply_migration(conn: &Connection, version: i64) -> Result<()> { 25 => apply_migration_025(conn), 26 => apply_migration_026(conn), 27 => apply_migration_027(conn), + 28 => apply_migration_028(conn), _ => bail!("unsupported schema migration version {version}"), } } @@ -392,6 +393,18 @@ mod tests { "collect a payload's remaining references", "SELECT 1 FROM quota_observations o WHERE o.payload_hash = ?1".to_string(), ), + ( + "look up a session's archive titles", + "SELECT title FROM archive_conversations WHERE session_hash IN (?1, ?2)" + .to_string(), + ), + ( + "look up a session's task-span titles", + "SELECT title FROM task_spans + WHERE is_meta = 0 + AND (session_hash IN (?1, ?2) OR session_uuid_hash IN (?1, ?2))" + .to_string(), + ), ] { let plan = query_plan(&conn, &sql); assert!( @@ -510,6 +523,68 @@ mod tests { )); } + #[test] + fn version_twenty_seven_backfills_session_title_keys() { + let conn = Connection::open_in_memory().expect("open in-memory database"); + ensure_migrations_table(&conn).expect("ensure migrations table"); + for version in 1..=27 { + apply_migration(&conn, version).expect("apply earlier migration"); + record_migration(&conn, version).expect("record earlier migration"); + } + let rollout = + "sessions/2026/06/01/rollout-2026-06-01T08-00-00-0199a0b1-c2d3-7e4f-8a9b-0c1d2e3f4a5b"; + conn.execute_batch(&format!( + r#" + INSERT INTO task_spans (span_id, provider, source_id, project_bucket, started_at, + title, normalized_title, confidence, payload) + VALUES + ('span-rollout', 'codex', 'source-1', 'bucket', '2026-06-01T08:00:00Z', + 'Fix parser', 'fix parser', 'high', '{{"session_id":"{rollout}"}}'), + ('span-none', 'codex', 'source-1', 'bucket', '2026-06-01T08:00:00Z', + 'Other', 'other', 'high', '{{}}'); + INSERT INTO archive_conversations (conversation_id, provider, source_id, + native_conversation_id, completeness, imported_at) + VALUES ('conversation-1', 'claude_code', 'source-2', 'claude-session', 'complete', + '2026-06-01T08:00:00Z'); + "# + )) + .expect("seed version twenty-seven rows"); + + migrate(&conn).expect("migrate version twenty-seven database"); + + let span_keys = |span_id: &str| { + conn.query_row( + "SELECT session_hash, session_uuid_hash FROM task_spans WHERE span_id = ?1", + [span_id], + |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, Option>(1)?, + )) + }, + ) + .expect("span keys") + }; + assert_eq!( + span_keys("span-rollout"), + ( + Some(statsai_core::hash_text(rollout)), + Some(statsai_core::hash_text( + "0199a0b1-c2d3-7e4f-8a9b-0c1d2e3f4a5b" + )), + ) + ); + assert_eq!(span_keys("span-none"), (None, None)); + let archive_key: String = conn + .query_row( + "SELECT session_hash FROM archive_conversations WHERE conversation_id = 'conversation-1'", + [], + |row| row.get(0), + ) + .expect("archive key"); + assert_eq!(archive_key, statsai_core::hash_text("claude-session")); + } + #[test] fn version_eleven_archive_receives_source_record_index() { let conn = Connection::open_in_memory().expect("open in-memory database"); diff --git a/crates/statsai-store/src/migrations/v2.rs b/crates/statsai-store/src/migrations/v2.rs index a4c8b46..1b64771 100644 --- a/crates/statsai-store/src/migrations/v2.rs +++ b/crates/statsai-store/src/migrations/v2.rs @@ -594,3 +594,64 @@ pub(crate) fn apply_migration_027(conn: &Connection) -> Result<()> { )?; Ok(()) } + +/// Hashed session keys on task spans and archive conversations, so a session +/// refresh looks up the titles of its own sessions instead of reading both +/// tables whole. Session names also record the parent a sub-agent borrows its +/// name from. +pub(crate) fn apply_migration_028(conn: &Connection) -> Result<()> { + conn.execute_batch( + r#" + ALTER TABLE task_spans ADD COLUMN session_hash TEXT; + ALTER TABLE task_spans ADD COLUMN session_uuid_hash TEXT; + ALTER TABLE archive_conversations ADD COLUMN session_hash TEXT; + ALTER TABLE session_names ADD COLUMN parent_hash TEXT; + CREATE INDEX IF NOT EXISTS session_names_parent_idx + ON session_names (parent_hash) WHERE parent_hash IS NOT NULL; + "#, + )?; + let spans = { + let mut statement = conn.prepare( + "SELECT span_id, json_extract(payload, '$.session_id') FROM task_spans + WHERE json_extract(payload, '$.session_id') IS NOT NULL", + )?; + let rows = statement.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })?; + rows.collect::>>()? + }; + let mut update_span = conn.prepare( + "UPDATE task_spans SET session_hash = ?2, session_uuid_hash = ?3 WHERE span_id = ?1", + )?; + for (span_id, session_id) in spans { + let (session_hash, session_uuid_hash) = crate::task_span_session_hashes(Some(&session_id)); + update_span.execute(params![span_id, session_hash, session_uuid_hash])?; + } + let conversations = { + let mut statement = conn + .prepare("SELECT conversation_id, native_conversation_id FROM archive_conversations")?; + let rows = statement.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })?; + rows.collect::>>()? + }; + let mut update_conversation = conn + .prepare("UPDATE archive_conversations SET session_hash = ?2 WHERE conversation_id = ?1")?; + for (conversation_id, native_id) in conversations { + update_conversation.execute(params![ + conversation_id, + statsai_core::hash_text(&native_id) + ])?; + } + conn.execute_batch( + r#" + CREATE INDEX IF NOT EXISTS task_spans_session_hash_idx + ON task_spans (session_hash) WHERE session_hash IS NOT NULL; + CREATE INDEX IF NOT EXISTS task_spans_session_uuid_hash_idx + ON task_spans (session_uuid_hash) WHERE session_uuid_hash IS NOT NULL; + CREATE INDEX IF NOT EXISTS archive_conversations_session_hash_idx + ON archive_conversations (session_hash); + "#, + )?; + Ok(()) +} diff --git a/crates/statsai-store/src/pricing/tests/repricing.rs b/crates/statsai-store/src/pricing/tests/repricing.rs index c6131ca..266b670 100644 --- a/crates/statsai-store/src/pricing/tests/repricing.rs +++ b/crates/statsai-store/src/pricing/tests/repricing.rs @@ -384,7 +384,7 @@ fn daily_rollups_are_unused_by_report_and_sync_paths() { assert_eq!(events[0].cost, expected_review_cost(started_at)); // Pin: bump this when CURRENT_SCHEMA_VERSION changes, after confirming // the daily_rollups table is still unused by report/sync/snapshot. - assert_eq!(CURRENT_SCHEMA_VERSION, 27); + assert_eq!(CURRENT_SCHEMA_VERSION, 28); } #[test] diff --git a/crates/statsai-store/src/rollups/mod.rs b/crates/statsai-store/src/rollups/mod.rs index 5407b84..38807a1 100644 --- a/crates/statsai-store/src/rollups/mod.rs +++ b/crates/statsai-store/src/rollups/mod.rs @@ -4,6 +4,7 @@ mod build; mod sessions; mod summaries; +pub(crate) use sessions::task_span_session_hashes; pub use sessions::{SessionFilter, SessionSort, SessionStats}; pub(crate) use build::*; diff --git a/crates/statsai-store/src/rollups/sessions.rs b/crates/statsai-store/src/rollups/sessions.rs index 431ff04..ac6745c 100644 --- a/crates/statsai-store/src/rollups/sessions.rs +++ b/crates/statsai-store/src/rollups/sessions.rs @@ -6,10 +6,10 @@ use anyhow::Result; use chrono::{DateTime, Utc}; use rusqlite::{params, OptionalExtension}; use statsai_core::{ - hash_text, project_has_stable_identity, provider_session_title_is_unusable, - task_title_is_generic, Confidence, CostAccumulator, CostInfo, ModelInfo, SessionName, - SessionRollupV1, SessionTitleSource, SummaryModelUsage, UsageCounts, UsageEvent, - SESSION_ROLLUP_SCHEMA_VERSION, + hash_text, project_has_stable_identity, provider_session_name, + provider_session_title_is_unusable, task_title_is_generic, Confidence, CostAccumulator, + CostInfo, ModelInfo, SessionName, SessionRollupV1, SessionTitleSource, SummaryModelUsage, + UsageCounts, UsageEvent, SESSION_ROLLUP_SCHEMA_VERSION, }; use std::collections::{BTreeMap, BTreeSet}; @@ -189,7 +189,8 @@ impl Store { } /// Records the names providers hold for sessions and refreshes the sessions - /// whose name changed. Unchanged names write nothing. + /// whose name changed, with the sub-agents that borrow it. Unchanged names + /// write nothing. A session's own name is never replaced by a borrowed one. pub fn upsert_session_names(&self, names: &[SessionName]) -> Result { if names.is_empty() { return Ok(0); @@ -198,21 +199,41 @@ impl Store { { let mut statement = self.conn.prepare( r#" - INSERT INTO session_names (local_session_id_hash, title, updated_at) - VALUES (?1, ?2, ?3) + INSERT INTO session_names (local_session_id_hash, title, parent_hash, updated_at) + VALUES (?1, ?2, ?3, ?4) ON CONFLICT(local_session_id_hash) DO UPDATE SET title = excluded.title, + parent_hash = excluded.parent_hash, updated_at = excluded.updated_at - WHERE session_names.title IS NOT excluded.title + WHERE (session_names.title IS NOT excluded.title + OR session_names.parent_hash IS NOT excluded.parent_hash) + AND (excluded.parent_hash IS NULL OR session_names.parent_hash IS NOT NULL) "#, )?; let now = Utc::now().to_rfc3339(); for name in names { - if statement.execute(params![&name.local_session_id_hash, &name.title, &now])? > 0 { + if statement.execute(params![ + &name.local_session_id_hash, + &name.title, + &name.parent_local_session_id_hash, + &now + ])? > 0 + { changed_hashes.push(name.local_session_id_hash.clone()); } } } + for chunk in changed_hashes.clone().chunks(500) { + let placeholders = sqlite_in_clause_placeholders(chunk.len()); + let mut statement = self.conn.prepare(&format!( + "SELECT local_session_id_hash FROM session_names WHERE parent_hash IN ({placeholders})" + ))?; + let params = sqlite_string_params(chunk); + let rows = statement.query_map(params.as_slice(), |row| row.get::<_, String>(0))?; + for row in rows { + changed_hashes.push(row?); + } + } let session_ids = self.session_ids_for_local_hashes(&changed_hashes)?; self.refresh_session_rollups_for_keys(&session_ids)?; Ok(changed_hashes.len() as u64) @@ -455,83 +476,108 @@ impl Store { &self, hashes: &[String], ) -> Result> { - let wanted = hashes.iter().map(String::as_str).collect::>(); + let wanted = hashes + .iter() + .map(String::as_str) + .collect::>() + .into_iter() + .collect::>(); + let mut titles = BTreeMap::new(); if wanted.is_empty() { - return Ok(BTreeMap::new()); + return Ok(titles); } - let mut titles = BTreeMap::new(); - let mut archive = self.conn.prepare( - "SELECT native_conversation_id, title - FROM archive_conversations - WHERE title IS NOT NULL - ORDER BY updated_at, conversation_id", - )?; - let archive_rows = archive.query_map([], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)) - })?; - for row in archive_rows { - let (native_id, title) = row?; - remember_hashed_title( + // Rows are gathered across chunks and then ordered, so the latest title + // wins no matter which chunk held it. + let mut archive_rows = Vec::new(); + let mut span_rows = Vec::new(); + let mut name_rows = Vec::new(); + for chunk in wanted.chunks(300) { + let placeholders = sqlite_in_clause_placeholders(chunk.len()); + let params = chunk + .iter() + .map(|hash| hash as &dyn rusqlite::types::ToSql) + .collect::>(); + let mut archive = self.conn.prepare(&format!( + "SELECT updated_at, conversation_id, session_hash, title + FROM archive_conversations + WHERE session_hash IN ({placeholders}) AND title IS NOT NULL" + ))?; + for row in archive.query_map(params.as_slice(), |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + )) + })? { + archive_rows.push(row?); + } + // A Codex span may name its session by rollout path; the events + // carry the thread id inside it. + let mut spans = self.conn.prepare(&format!( + "SELECT started_at, span_id, + CASE WHEN session_hash IN ({placeholders}) THEN session_hash + ELSE session_uuid_hash END, + title + FROM task_spans + WHERE is_meta = 0 + AND (session_hash IN ({placeholders}) + OR session_uuid_hash IN ({placeholders}))" + ))?; + let mut span_params = params.clone(); + span_params.extend(params.iter().copied()); + span_params.extend(params.iter().copied()); + for row in spans.query_map(span_params.as_slice(), |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, Option>(3)?, + )) + })? { + span_rows.push(row?); + } + // A borrowed name reads the parent's own name as it is now. + let mut names = self.conn.prepare(&format!( + "SELECT child.local_session_id_hash, + CASE WHEN child.parent_hash IS NULL THEN child.title + WHEN parent.title IS NOT NULL + THEN parent.title || ' · ' || child.title + END + FROM session_names child + LEFT JOIN session_names parent + ON parent.local_session_id_hash = child.parent_hash + AND parent.parent_hash IS NULL + WHERE child.local_session_id_hash IN ({placeholders})" + ))?; + for row in names.query_map(params.as_slice(), |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)) + })? { + name_rows.push(row?); + } + } + archive_rows.sort(); + for (_, _, hash, title) in archive_rows { + remember_session_title(&mut titles, hash, Some(&title), SessionTitleSource::Archive); + } + span_rows.sort(); + for (_, _, hash, title) in span_rows { + remember_session_title( &mut titles, - &wanted, - &native_id, + hash, title.as_deref(), - SessionTitleSource::Archive, + SessionTitleSource::TaskSpan, ); } - drop(archive); - - let mut spans = self.conn.prepare( - "SELECT json_extract(payload, '$.session_id'), - json_extract(payload, '$.title'), - json_extract(payload, '$.is_meta') - FROM task_spans - ORDER BY started_at, span_id", - )?; - let span_rows = spans.query_map([], |row| { - Ok(( - row.get::<_, Option>(0)?, - row.get::<_, Option>(1)?, - row.get::<_, Option>(2)?, - )) - })?; - for row in span_rows { - let (session_id, title, is_meta) = row?; - if is_meta.unwrap_or(0) != 0 { - continue; - } - let Some(session_id) = session_id.filter(|value| !value.trim().is_empty()) else { - continue; - }; - let raw_ids = - std::iter::once(session_id.as_str()).chain(codex_rollout_session_uuid(&session_id)); - for raw_id in raw_ids { - remember_hashed_title( - &mut titles, - &wanted, - raw_id, - title.as_deref(), - SessionTitleSource::TaskSpan, - ); - } - } - drop(spans); - // A provider-held name outranks task and archive titles for its session. - let mut names = self - .conn - .prepare("SELECT local_session_id_hash, title FROM session_names")?; - let name_rows = names.query_map([], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) - })?; - for row in name_rows { - let (hash, title) = row?; - if !wanted.contains(hash.as_str()) { - continue; - } - if let Some(title) = acceptable_session_title(Some(&title), SessionTitleSource::Event) { - titles.insert(hash, (title, SessionTitleSource::Event)); - } + for (hash, title) in name_rows { + let title = provider_session_name(title.as_deref(), 90); + remember_session_title( + &mut titles, + hash, + title.as_deref(), + SessionTitleSource::Event, + ); } Ok(titles) } @@ -858,21 +904,29 @@ fn acceptable_session_title(value: Option<&str>, source: SessionTitleSource) -> (!trimmed.is_empty() && !rejected).then(|| trimmed.to_string()) } -fn remember_hashed_title( +fn remember_session_title( titles: &mut BTreeMap, - wanted: &BTreeSet<&str>, - raw_id: &str, + hash: String, title: Option<&str>, source: SessionTitleSource, ) { - let Some(title) = acceptable_session_title(title, source) else { - return; - }; - let hash = hash_text(raw_id); - if !wanted.contains(hash.as_str()) { - return; + if let Some(title) = acceptable_session_title(title, source) { + titles.insert(hash, (title, source)); } - titles.insert(hash, (title, source)); +} + +/// The hashed keys a task span's session is looked up by: its session id, and +/// for a Codex rollout path, the thread id inside it. +pub(crate) fn task_span_session_hashes( + session_id: Option<&str>, +) -> (Option, Option) { + let Some(session_id) = session_id.filter(|value| !value.trim().is_empty()) else { + return (None, None); + }; + ( + Some(hash_text(session_id)), + codex_rollout_session_uuid(session_id).map(hash_text), + ) } fn session_rollup_content_hash(rollup: &SessionRollupV1) -> Result { diff --git a/crates/statsai-store/src/sync_state/pending.rs b/crates/statsai-store/src/sync_state/pending.rs index 90b960d..8c38c2b 100644 --- a/crates/statsai-store/src/sync_state/pending.rs +++ b/crates/statsai-store/src/sync_state/pending.rs @@ -483,6 +483,8 @@ impl Store { ¤t_account_evidence_summaries, )?; let current_snapshot = self.current_http_sync_authoritative_snapshot( + target, + include_projects, ¤t_rollups, ¤t_passthrough_summaries, ¤t_code_change_metrics, @@ -525,8 +527,11 @@ impl Store { }) } + #[allow(clippy::too_many_arguments)] pub(crate) fn current_http_sync_authoritative_snapshot( &self, + target: &str, + include_projects: bool, rollups: &[UsageSummary], passthrough_summaries: &[UsageSummary], code_change_metrics: &[CodeChangeMetric], @@ -586,11 +591,45 @@ impl Store { .into_iter() .map(|coverage| coverage.coverage_id) .collect(), - session_rollup_ids: if self.sync_preferences()?.include_sessions { - Some(self.session_rollup_ids()?) - } else { - None - }, + session_rollup_ids: self.authoritative_session_rollup_ids( + "http", + target, + self.sync_preferences()?.include_sessions, + include_projects, + || self.session_rollup_ids(), + )?, }) } + + /// The session ids an authoritative snapshot names for a target. + /// + /// With session sync on, every local session. With projects excluded, an + /// empty list while the target still holds sessions from this device: each + /// carries its project, path, repository and branch, so excluding projects + /// retires them. With only session sync off, nothing, and hosted sessions + /// stay until it is turned back on. + pub fn authoritative_session_rollup_ids( + &self, + sink: &str, + target: &str, + include_sessions: bool, + include_projects: bool, + session_ids: impl FnOnce() -> Result>, + ) -> Result>> { + if include_sessions { + return session_ids().map(Some); + } + if include_projects { + return Ok(None); + } + let tracked = self.conn.query_row( + "SELECT EXISTS( + SELECT 1 FROM entity_sync_state + WHERE sink = ?1 AND target = ?2 AND entity_kind = 'session_rollup' + )", + params![sink, target], + |row| row.get::<_, bool>(0), + )?; + Ok(tracked.then(Vec::new)) + } } diff --git a/crates/statsai-store/src/tasks/spans.rs b/crates/statsai-store/src/tasks/spans.rs index ebcea8f..8e81dfa 100644 --- a/crates/statsai-store/src/tasks/spans.rs +++ b/crates/statsai-store/src/tasks/spans.rs @@ -82,9 +82,10 @@ impl Store { span_id, provider, source_id, project_bucket, started_at, ended_at, title, normalized_title, is_meta, confidence, source_file_path_hash, event_count, has_usage_evidence, total_messages, user_messages, assistant_messages, - developer_messages, payload + developer_messages, payload, session_hash, session_uuid_hash ) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, + ?19, ?20) ON CONFLICT(span_id) DO UPDATE SET provider = excluded.provider, source_id = excluded.source_id, @@ -102,7 +103,9 @@ impl Store { user_messages = excluded.user_messages, assistant_messages = excluded.assistant_messages, developer_messages = excluded.developer_messages, - payload = excluded.payload + payload = excluded.payload, + session_hash = excluded.session_hash, + session_uuid_hash = excluded.session_uuid_hash "#, )?; let mut delete_links = self @@ -117,6 +120,8 @@ impl Store { )?; for span in spans { let payload = serde_json::to_string(span)?; + let (session_hash, session_uuid_hash) = + crate::task_span_session_hashes(span.session_id.as_deref()); changed += span_stmt.execute(params![ &span.span_id.0, &span.provider, @@ -136,6 +141,8 @@ impl Store { safe_u64_to_i64(span.assistant_messages), safe_u64_to_i64(span.developer_messages), &payload, + session_hash, + session_uuid_hash, ])? as u64; delete_links.execute(params![&span.span_id.0])?; for event_id in &span.linked_event_ids { diff --git a/crates/statsai-store/src/tests/rollups/sessions.rs b/crates/statsai-store/src/tests/rollups/sessions.rs index 22aa5eb..ebfe4c0 100644 --- a/crates/statsai-store/src/tests/rollups/sessions.rs +++ b/crates/statsai-store/src/tests/rollups/sessions.rs @@ -1046,6 +1046,7 @@ fn a_provider_name_renames_a_stored_session_without_its_events() { let rename = |title: &str| statsai_core::SessionName { local_session_id_hash: hash_text("named-session"), title: title.to_string(), + parent_local_session_id_hash: None, }; assert_eq!( store @@ -1065,3 +1066,59 @@ fn a_provider_name_renames_a_stored_session_without_its_events() { 0 ); } + +#[test] +fn a_sub_agent_borrows_its_parents_current_name() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/session-parent-names"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let start = Utc + .with_ymd_and_hms(2026, 6, 19, 9, 0, 0) + .single() + .expect("start"); + let mut event = test_store_event(&source, start, "child"); + stamp_session(&mut event, "child-thread"); + store.insert_event(&event).expect("insert"); + + let name = |raw: &str, title: &str, parent: Option<&str>| statsai_core::SessionName { + local_session_id_hash: hash_text(raw), + title: title.to_string(), + parent_local_session_id_hash: parent.map(hash_text), + }; + let title = || { + store.all_session_rollups().expect("sessions")[0] + .title + .clone() + }; + store + .upsert_session_names(&[ + name("parent-thread", "Design tool/plugin", None), + name("child-thread", "Popper", Some("parent-thread")), + ]) + .expect("names"); + assert_eq!(title().as_deref(), Some("Design tool/plugin · Popper")); + + // Renaming the parent renames the sub-agent, whose rollout is not re-read. + store + .upsert_session_names(&[name("parent-thread", "Plugin API", None)]) + .expect("rename parent"); + assert_eq!(title().as_deref(), Some("Plugin API · Popper")); + + // A name of its own wins, and a borrowed one does not replace it. + store + .upsert_session_names(&[name("child-thread", "Review plugin", None)]) + .expect("own name"); + assert_eq!( + store + .upsert_session_names(&[name("child-thread", "Popper", Some("parent-thread"))]) + .expect("borrowed again"), + 0 + ); + assert_eq!(title().as_deref(), Some("Review plugin")); +} diff --git a/crates/statsai/src/cli/args/sync.rs b/crates/statsai/src/cli/args/sync.rs index 6001a5e..18a5b79 100644 --- a/crates/statsai/src/cli/args/sync.rs +++ b/crates/statsai/src/cli/args/sync.rs @@ -56,7 +56,7 @@ pub(crate) struct SyncCommand { #[arg( long, conflicts_with_all = ["include_projects", "include_tasks", "include_sessions"], - help = "Disable project metadata sync for this device and future syncs" + help = "Disable project metadata sync for this device and future syncs; hosted sessions are retired" )] pub(crate) exclude_projects: bool, #[arg( diff --git a/crates/statsai/src/cli/sync/batch.rs b/crates/statsai/src/cli/sync/batch.rs index f8be41e..69367d2 100644 --- a/crates/statsai/src/cli/sync/batch.rs +++ b/crates/statsai/src/cli/sync/batch.rs @@ -350,12 +350,18 @@ pub(crate) fn build_sync_batch_with_identity_key( .iter() .map(|coverage| coverage.coverage_id.clone()) .collect(), - session_rollup_ids: sync_preferences.include_sessions.then(|| { - all_sessions - .iter() - .map(|rollup| rollup.session_id.clone()) - .collect() - }), + session_rollup_ids: store.authoritative_session_rollup_ids( + &command.sink, + target, + sync_preferences.include_sessions, + include_projects, + || { + Ok(all_sessions + .iter() + .map(|rollup| rollup.session_id.clone()) + .collect()) + }, + )?, }; let failed_without_resume = state.as_ref().is_some_and(|state| { state.failure_count > 0 && state.pending_resume_batch_id.is_none() diff --git a/crates/statsai/src/cli/sync/mod.rs b/crates/statsai/src/cli/sync/mod.rs index 81837c7..d65e291 100644 --- a/crates/statsai/src/cli/sync/mod.rs +++ b/crates/statsai/src/cli/sync/mod.rs @@ -120,7 +120,11 @@ fn apply_sync_preference_overrides( "session sync enabled: the next sync backfills session rollups, and later snapshots retire hosted sessions this device no longer has" ); } - if original.include_sessions && !preferences.include_sessions { + if original.include_projects && !preferences.include_projects { + eprintln!( + "project sync disabled: the next successful HTTP sync retires this device's hosted sessions, which carry project names and paths" + ); + } else if original.include_sessions && !preferences.include_sessions { eprintln!( "session sync disabled: hosted sessions stay in place until session sync is enabled again" ); diff --git a/crates/statsai/src/cli/tests/sync/batch.rs b/crates/statsai/src/cli/tests/sync/batch.rs index 5905fd5..efe7405 100644 --- a/crates/statsai/src/cli/tests/sync/batch.rs +++ b/crates/statsai/src/cli/tests/sync/batch.rs @@ -2133,3 +2133,70 @@ fn activity_sync_payload_omits_paths_commands_and_invocation_ids() { assert!(excluded.get("activity_rollups").is_none()); assert!(excluded.get("activity_coverage").is_none()); } + +#[test] +fn excluding_projects_retires_hosted_sessions_and_excluding_sessions_keeps_them() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/codex-sessions-exclude-projects"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + store + .insert_event(&test_event( + "codex", + &source, + Utc::now(), + None, + TokenParts::total(15), + )) + .expect("event"); + let command = |exclude_projects: bool, exclude_sessions: bool| SyncCommand { + exclude_projects, + exclude_sessions, + dry_run: true, + endpoint: Some("https://api.example.com/api/sync/batches".to_string()), + ..test_sync_command("http") + }; + let target = sync_target(&command(false, false)).expect("target"); + let snapshot_session_ids = |command: &SyncCommand| { + build_sync_batch(command, &store, "device", &target) + .expect("batch") + .0 + .authoritative_snapshot + .and_then(|snapshot| snapshot.session_rollup_ids) + }; + + // Nothing hosted yet: excluding projects has no sessions to retire. + assert_eq!(snapshot_session_ids(&command(true, false)), None); + + let now = Utc + .with_ymd_and_hms(2026, 6, 20, 12, 0, 0) + .single() + .expect("date"); + let mut synced = test_task_only_sync_batch(now, 0, 0); + synced.batch_id = "batch_sessions".to_string(); + synced.sessions = test_session_rollups(2); + let logical_batch_id = logical_http_rollup_batch_id(&synced.batch_id).to_string(); + record_rollup_sync_chunk_success(&store, "http", &target, &logical_batch_id, &synced) + .expect("record synced sessions"); + + // Hosted sessions carry their project, so excluding projects retires them. + assert_eq!( + snapshot_session_ids(&command(true, false)), + Some(Vec::new()) + ); + // Turning off only session sync leaves them in place. + store + .set_sync_preferences(SyncPreferences { + include_projects: true, + include_tasks: false, + include_activity: false, + include_sessions: true, + }) + .expect("sessions on"); + assert_eq!(snapshot_session_ids(&command(false, true)), None); +} diff --git a/crates/statsai/tests/cli_surface/help.txt b/crates/statsai/tests/cli_surface/help.txt index 4a29a61..d240415 100644 --- a/crates/statsai/tests/cli_surface/help.txt +++ b/crates/statsai/tests/cli_surface/help.txt @@ -1053,7 +1053,7 @@ Options: --yes Confirm destructive sync reset actions --dry-run Build the sync batch without writing --include-projects Enable project metadata sync for this device and future syncs - --exclude-projects Disable project metadata sync for this device and future syncs + --exclude-projects Disable project metadata sync for this device and future syncs; hosted sessions are retired --include-tasks Enable hosted task sync for this device and future syncs (implies --include-projects) --exclude-tasks Disable hosted task sync for this device and future syncs --include-activity Enable hosted agent activity sync for this device and future syncs diff --git a/docs/sync-contract.md b/docs/sync-contract.md index 8c19b48..8dd6241 100644 --- a/docs/sync-contract.md +++ b/docs/sync-contract.md @@ -37,7 +37,9 @@ and are not sent, and neither is a session with no project and no title. Prompts the device. Session retirement is not implied by `sync_batch.v6`. A snapshot retires hosted sessions only when it includes `session_rollup_ids`. An empty array prunes them. Omitting the key leaves hosted sessions in place, which is -what the collector does when `include_sessions` is off. Pre-v6 acknowledgements +what the collector does when `include_sessions` alone is off. With projects +excluded, it sends the empty array until the target holds none of its sessions, +since each hosted session carries a project. Pre-v6 acknowledgements omit the `sessions` counter. The collector owns local scanning, normalization, idempotent local storage, and privacy scrubbing. The backend owns authentication, validation, deduplication,