From a5b8fbc91072b49f5251c7825e091c70cee6bda0 Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:49:57 +0300 Subject: [PATCH 01/13] WIP: Codex account plan and subscription evidence Uncommitted collector/store/sync work for account-plan evidence, shelved so the quota-cycle stack (PR #13) can be tested locally. Not reviewed for merge: carries a sync_batch.v4 definition that conflicts with PR #13. --- Cargo.lock | 1 + README.md | 2 +- crates/statsai-adapters/Cargo.toml | 1 + crates/statsai-adapters/src/lib.rs | 1270 +++++++++++++++++- crates/statsai-core/src/account_plan.rs | 315 +++++ crates/statsai-core/src/lib.rs | 39 +- crates/statsai-daemon/src/lib.rs | 334 ++++- crates/statsai-store/src/account_plan.rs | 1197 +++++++++++++++++ crates/statsai-store/src/code_changes.rs | 2 + crates/statsai-store/src/lib.rs | 701 +++++++++- crates/statsai-store/src/migrations.rs | 107 +- crates/statsai-store/src/quota.rs | 96 +- crates/statsai-store/tests/forward_schema.rs | 20 +- crates/statsai-sync/src/lib.rs | 48 +- crates/statsai/src/main.rs | 1195 +++++++++++++++- docs/sync-contract.md | 45 +- docs/task-collection.md | 2 +- 17 files changed, 5184 insertions(+), 191 deletions(-) create mode 100644 crates/statsai-core/src/account_plan.rs create mode 100644 crates/statsai-store/src/account_plan.rs diff --git a/Cargo.lock b/Cargo.lock index 3899396..ffdd553 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2421,6 +2421,7 @@ dependencies = [ "anyhow", "base64", "chrono", + "regex", "rusqlite", "serde", "serde_json", diff --git a/README.md b/README.md index 78ed69b..6eb8431 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ dashboard. Raw usage events and complete archived conversation records stay local and are never included in hosted sync. StatsAI does not upload full prompts, full responses, or raw provider logs. When hosted task sync is explicitly enabled -with `statsai sync --include-tasks`, the current `sync_batch.v3` payload may +with `statsai sync --include-tasks`, the current `sync_batch.v4` payload may include bounded conversation-derived task titles, summary previews, and todo excerpts. You can inspect the exact sync contract in [`docs/sync-contract.md`](docs/sync-contract.md) and verify the resolved sync diff --git a/crates/statsai-adapters/Cargo.toml b/crates/statsai-adapters/Cargo.toml index 885b813..7ed3bac 100644 --- a/crates/statsai-adapters/Cargo.toml +++ b/crates/statsai-adapters/Cargo.toml @@ -20,6 +20,7 @@ anyhow.workspace = true base64.workspace = true chrono.workspace = true rusqlite.workspace = true +regex.workspace = true serde.workspace = true serde_json.workspace = true url.workspace = true diff --git a/crates/statsai-adapters/src/lib.rs b/crates/statsai-adapters/src/lib.rs index 0a071ab..03ea70a 100644 --- a/crates/statsai-adapters/src/lib.rs +++ b/crates/statsai-adapters/src/lib.rs @@ -2,22 +2,29 @@ use anyhow::{Context, Result}; use chrono::{DateTime, NaiveDate, TimeZone, Utc}; +use regex::Regex; use rusqlite::{Connection, OpenFlags}; use serde::Deserialize; use serde_json::Value; use statsai_core::{ - branch_family, canonical_display, display_path, expand_home_path, extract_issue_keys, - hash_text, home_dir, normalize_git_remote, normalize_task_title, path_hash, project_bucket_key, - semantic_event_id, summarize_task_text, summary_id, task_preview_from_prompt, task_span_id, + account_identity_observation_id, account_plan_observation_id, branch_family, canonical_display, + conversation_account_binding_id, display_path, expand_home_path, extract_issue_keys, hash_text, + home_dir, normalize_email, normalize_git_remote, normalize_plan_name, normalize_task_title, + path_hash, project_bucket_key, provider_account_id_from_identity, semantic_event_id, + summarize_task_text, summary_id, task_preview_from_prompt, task_span_id, task_title_from_prompt, task_title_is_generic, task_title_is_weak_signal, - task_title_signal_score, title_topic_tokens, BillingPeriod, Confidence, CostAccumulator, - CostInfo, EventId, EventSource, IdentitySource, LatencySource, LocationOrigin, MetricStats, - ModelInfo, ParseEvidence, PrivacyInfo, PrivacyMode, ProjectInfo, QuotaCreditsV1, - QuotaObservationRecordV1, QuotaObservationV1, QuotaStatusV1, QuotaUsageLinkKind, - QuotaWindowObservationV1, ReasoningLevel, RuntimeInfo, SessionInfo, SourceKind, SourceLocation, - SubscriptionStatus, SummaryMetadata, SummaryMetrics, TaskSpan, UsageCounts, UsageEvent, - UsageSummary, QUOTA_OBSERVATION_SCHEMA_VERSION, QUOTA_WINDOW_OBSERVATION_SCHEMA_VERSION, - TASK_SPAN_SCHEMA_VERSION, USAGE_EVENT_SCHEMA_VERSION, USAGE_SUMMARY_SCHEMA_VERSION, + task_title_signal_score, title_topic_tokens, AccountEvidenceCheckpointV1, AccountEvidenceKind, + AccountIdentityObservationV1, AccountPlanObservationV1, Confidence, + ConversationAccountBindingV1, CostAccumulator, CostInfo, EventId, EventSource, IdentitySource, + LatencySource, LocationOrigin, MetricStats, ModelInfo, ParseEvidence, PrivacyInfo, PrivacyMode, + ProjectInfo, ProviderAccountId, QuotaCreditsV1, QuotaObservationRecordV1, QuotaObservationV1, + QuotaStatusV1, QuotaUsageLinkKind, QuotaWindowObservationV1, ReasoningLevel, RuntimeInfo, + SessionInfo, SourceKind, SourceLocation, SummaryMetadata, SummaryMetrics, TaskSpan, + UsageCounts, UsageEvent, UsageSummary, ACCOUNT_EVIDENCE_CHECKPOINT_SCHEMA_VERSION, + ACCOUNT_IDENTITY_OBSERVATION_SCHEMA_VERSION, ACCOUNT_PLAN_OBSERVATION_SCHEMA_VERSION, + CONVERSATION_ACCOUNT_BINDING_SCHEMA_VERSION, QUOTA_OBSERVATION_SCHEMA_VERSION, + QUOTA_WINDOW_OBSERVATION_SCHEMA_VERSION, TASK_SPAN_SCHEMA_VERSION, USAGE_EVENT_SCHEMA_VERSION, + USAGE_SUMMARY_SCHEMA_VERSION, }; use statsai_pricing::{ estimate_cost_at, normalize_model_name, pricing_changes_between, unknown_cost, @@ -60,8 +67,86 @@ const CLAUDE_SETTINGS_AUTH_OVERRIDE_KEYS: &[&str] = &[ "CLAUDE_CODE_USE_ANTHROPIC_AWS", ]; const CODEX_TASK_PREVIEW_RAW_BYTES: usize = 24 * 1024; +const CODEX_ACCOUNT_EVIDENCE_PARSER_VERSION: &str = "codex-account-evidence.v1"; pub(crate) const MAX_JSONL_RECORD_BYTES: usize = 16 * 1024 * 1024; +#[derive(Debug, Clone, PartialEq, Eq)] +struct CodexTelemetryCursor { + maximum_row_id: i64, + checkpoint_row_fingerprint: Option, + database_size: i64, + database_modified_nanos: i64, + wal_size: i64, + wal_modified_nanos: i64, +} + +fn codex_telemetry_file_state( + path: &Path, + maximum_row_id: i64, + checkpoint_row_fingerprint: Option, +) -> CodexTelemetryCursor { + let metadata = std::fs::metadata(path).ok(); + let wal_metadata = std::fs::metadata(path.with_extension("sqlite-wal")).ok(); + let modified_nanos = |metadata: Option<&std::fs::Metadata>| { + metadata + .and_then(|value| value.modified().ok()) + .and_then(|value| value.duration_since(UNIX_EPOCH).ok()) + .map_or(0, |value| { + i64::try_from(value.as_nanos()).unwrap_or(i64::MAX) + }) + }; + CodexTelemetryCursor { + maximum_row_id, + checkpoint_row_fingerprint, + database_size: metadata + .as_ref() + .map_or(0, |value| i64::try_from(value.len()).unwrap_or(i64::MAX)), + database_modified_nanos: modified_nanos(metadata.as_ref()), + wal_size: wal_metadata + .as_ref() + .map_or(0, |value| i64::try_from(value.len()).unwrap_or(i64::MAX)), + wal_modified_nanos: modified_nanos(wal_metadata.as_ref()), + } +} + +fn codex_telemetry_cursor(checkpoint: &AccountEvidenceCheckpointV1) -> CodexTelemetryCursor { + CodexTelemetryCursor { + maximum_row_id: checkpoint.maximum_row_id, + checkpoint_row_fingerprint: checkpoint.checkpoint_row_fingerprint.clone(), + database_size: checkpoint.database_size, + database_modified_nanos: checkpoint.database_modified_nanos, + wal_size: checkpoint.wal_size, + wal_modified_nanos: checkpoint.wal_modified_nanos, + } +} + +fn codex_telemetry_checkpoint_row_fingerprint( + connection: &Connection, + row_id: i64, +) -> Option { + if row_id == 0 { + return Some(hash_text("codex-telemetry-checkpoint-row.v1:empty")); + } + let (seconds, nanos, body) = connection + .query_row( + "SELECT ts, ts_nanos, feedback_log_body FROM logs WHERE id = ?1", + [row_id], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, Option>(2)?, + )) + }, + ) + .ok()?; + Some(hash_text(&format!( + "codex-telemetry-checkpoint-row.v1:{row_id}:{seconds}:{nanos}:{}", + body.as_deref() + .map_or_else(|| "none".to_string(), hash_text,) + ))) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum BoundedLineRead { Eof, @@ -203,6 +288,111 @@ pub struct AdapterScan { pub verified_source_state: Option, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ObservedProviderAccount { + pub provider_user_id: Option, + pub email: Option, + pub plan_name: Option, + pub observed_at: DateTime, +} + +#[derive(Debug, Clone, Default)] +pub struct AccountEvidenceScan { + pub accounts: Vec, + pub identity_observations: Vec, + pub plan_observations: Vec, + pub conversation_bindings: Vec, + pub checkpoints: Vec, +} + +/// Rewrites collected account references and their account-dependent deterministic IDs. +/// +/// Callers use this with already-known aliases before Store deduplication, and again after an +/// identity upsert when a newly discovered identity changes the canonical account reference. +pub fn remap_account_evidence_account_ids( + evidence: &mut AccountEvidenceScan, + canonical_ids: &HashMap, +) { + for observation in &mut evidence.identity_observations { + if let Some(canonical_id) = observation + .provider_account_id + .as_ref() + .and_then(|account_id| canonical_ids.get(account_id)) + { + observation.provider_account_id = Some(canonical_id.clone()); + } + } + for observation in &mut evidence.plan_observations { + if let Some(canonical_id) = observation + .provider_account_id + .as_ref() + .and_then(|account_id| canonical_ids.get(account_id)) + { + observation.provider_account_id = Some(canonical_id.clone()); + observation.observation_id = account_plan_observation_id( + &observation.source_id, + Some(canonical_id), + &observation.raw_plan_name, + observation.observed_at, + observation.evidence_kind, + ); + } + } + for binding in &mut evidence.conversation_bindings { + if let Some(canonical_id) = canonical_ids.get(&binding.provider_account_id) { + binding.provider_account_id = canonical_id.clone(); + binding.binding_id = conversation_account_binding_id( + &binding.source_id, + &binding.conversation_id_hash, + binding.turn_id_hash.as_deref(), + canonical_id, + ); + } + } +} + +/// Drops discovered accounts whose evidence was already filtered out by the Store. +/// +/// Call this after `Store::retain_unseen_account_evidence` so unrelated usage-file changes do not +/// refresh an unchanged account and mark it pending for cloud sync again. +pub fn retain_accounts_referenced_by_account_evidence( + provider: &str, + canonical_ids: &HashMap, + evidence: &mut AccountEvidenceScan, +) { + let referenced_account_ids = evidence + .identity_observations + .iter() + .filter_map(|observation| observation.provider_account_id.clone()) + .chain( + evidence + .plan_observations + .iter() + .filter_map(|observation| observation.provider_account_id.clone()), + ) + .chain( + evidence + .conversation_bindings + .iter() + .map(|binding| binding.provider_account_id.clone()), + ) + .collect::>(); + evidence.accounts.retain(|observed| { + provider_account_id_from_identity( + provider, + observed.provider_user_id.as_deref(), + observed.email.as_deref(), + ) + .map(|account_id| { + canonical_ids + .get(&account_id) + .cloned() + .unwrap_or(account_id) + }) + .is_some_and(|account_id| referenced_account_ids.contains(&account_id)) + }); +} + pub trait ProviderAdapter: Send + Sync { fn id(&self) -> &'static str; fn version(&self) -> &'static str; @@ -237,6 +427,13 @@ pub trait ProviderAdapter: Send + Sync { ) -> bool { false } + fn collect_account_evidence( + &self, + _source: &SourceLocation, + _checkpoints: &[AccountEvidenceCheckpointV1], + ) -> Result { + Ok(AccountEvidenceScan::default()) + } fn scan(&self, source: &SourceLocation, options: &ScanOptions) -> Result; fn collect_archive( @@ -399,6 +596,17 @@ impl ProviderAdapter for CodexAdapter { .unwrap_or(VerifiedSourceObservation::Unavailable)) } + fn collect_account_evidence( + &self, + source: &SourceLocation, + checkpoints: &[AccountEvidenceCheckpointV1], + ) -> Result { + let Some(root) = source_root_path(source) else { + return Ok(AccountEvidenceScan::default()); + }; + collect_codex_account_evidence(source, &codex_source_root(&root), checkpoints) + } + fn scan(&self, source: &SourceLocation, options: &ScanOptions) -> Result { scan_codex_source(self, source, options) } @@ -5534,6 +5742,14 @@ fn open_sqlite_readonly(path: &Path) -> Result { .with_context(|| format!("open sqlite {}", path.display())) } +fn sqlite_table_exists(connection: &Connection, table: &str) -> Result { + Ok(connection.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1)", + [table], + |row| row.get(0), + )?) +} + fn sqlite_column_exists(connection: &Connection, table: &str, column: &str) -> Result { let mut statement = connection.prepare(&format!("PRAGMA table_info({table})"))?; let mut rows = statement.query([])?; @@ -7627,7 +7843,19 @@ fn claude_profile_timestamp(value: &Value) -> Option> { } } -fn codex_auth_snapshot(root: &Path) -> Option { +#[derive(Debug)] +struct CodexAuthClaims { + provider_user_id: Option, + email: Option, + plan_type: Option, + auth_mode: Option, + authenticated_at: Option>, + subscription_checked_at: Option>, + active_from: Option>, + active_until: Option>, +} + +fn codex_auth_claims(root: &Path) -> Option { let auth_path = root.join("auth.json"); let value = std::fs::read_to_string(&auth_path).ok()?; let value: Value = serde_json::from_str(&value).ok()?; @@ -7671,7 +7899,7 @@ fn codex_auth_snapshot(root: &Path) -> Option { } let plan_type = auth.and_then(|auth| string_at_any(auth, &["chatgpt_plan_type"])); - let plan_name = plan_type.as_deref().map(display_codex_plan_name); + let auth_mode = string_at_any(&value, &["auth_mode", "authMode"]); let authenticated_at = payload .as_ref() .and_then(|payload| timestamp_at_any(payload, &["auth_time", "iat"])) @@ -7680,28 +7908,646 @@ fn codex_auth_snapshot(root: &Path) -> Option { // that the embedded subscription claims were refreshed at the same time. let subscription_checked_at = auth.and_then(|auth| timestamp_at_any(auth, &["chatgpt_subscription_last_checked"])); - let verified_at = subscription_checked_at.or(authenticated_at); - let paid_at = + let active_from = auth.and_then(|auth| timestamp_at_any(auth, &["chatgpt_subscription_active_start"])); - let current_period_ends_at = + let active_until = auth.and_then(|auth| timestamp_at_any(auth, &["chatgpt_subscription_active_until"])); - let subscription = plan_type.as_deref().and_then(|plan_type| { - codex_verified_subscription( - plan_type, - paid_at, - current_period_ends_at, - subscription_checked_at, + Some(CodexAuthClaims { + provider_user_id, + email, + plan_type, + auth_mode, + authenticated_at, + subscription_checked_at, + active_from, + active_until, + }) +} + +fn collect_codex_account_evidence( + source: &SourceLocation, + root: &Path, + checkpoints: &[AccountEvidenceCheckpointV1], +) -> Result { + let mut scan = AccountEvidenceScan::default(); + collect_codex_auth_evidence(source, root, &mut scan); + collect_codex_telemetry_evidence(source, root, checkpoints, &mut scan)?; + collect_codex_reset_history_evidence(source, root, &mut scan); + collect_codex_login_evidence(source, root, &mut scan)?; + dedupe_codex_account_evidence(&mut scan); + Ok(scan) +} + +fn collect_codex_auth_evidence( + source: &SourceLocation, + root: &Path, + scan: &mut AccountEvidenceScan, +) { + let Some(claims) = codex_auth_claims(root) else { + return; + }; + let observed_at = claims + .subscription_checked_at + .or(claims.authenticated_at) + .unwrap_or_else(Utc::now); + let provider_account_id = provider_account_id_from_identity( + CODEX_PROVIDER, + claims.provider_user_id.as_deref(), + claims.email.as_deref(), + ); + let auth_path = root.join("auth.json"); + let artifact_path_hash = hash_text(&canonical_display(&auth_path)); + let record_fingerprint = hash_text(&format!( + "codex-auth-evidence.v1:{}:{}:{}:{}:{}:{}", + claims.provider_user_id.as_deref().unwrap_or("none"), + claims.email.as_deref().unwrap_or("none"), + claims.plan_type.as_deref().unwrap_or("none"), + claims.auth_mode.as_deref().unwrap_or("none"), + claims + .active_from + .map_or_else(|| "none".to_string(), |value| value.to_rfc3339()), + claims + .active_until + .map_or_else(|| "none".to_string(), |value| value.to_rfc3339()), + )); + scan.accounts.push(ObservedProviderAccount { + provider_user_id: claims.provider_user_id.clone(), + email: claims.email.clone(), + plan_name: claims.plan_type.as_deref().map(normalize_plan_name), + observed_at, + }); + scan.identity_observations + .push(AccountIdentityObservationV1 { + schema_version: ACCOUNT_IDENTITY_OBSERVATION_SCHEMA_VERSION.to_string(), + observation_id: account_identity_observation_id( + &source.source_id, + AccountEvidenceKind::AuthSnapshot, + observed_at, + &record_fingerprint, + ), + provider: CODEX_PROVIDER.to_string(), + source_id: source.source_id.clone(), + provider_account_id: provider_account_id.clone(), + provider_user_id_hash: claims.provider_user_id.as_deref().map(hash_text), + email_hash: claims + .email + .as_deref() + .map(normalize_email) + .as_deref() + .map(hash_text), + conversation_id_hash: None, + turn_id_hash: None, + observed_at, + evidence_kind: AccountEvidenceKind::AuthSnapshot, + confidence: Confidence::High, + auth_mode: claims.auth_mode.clone(), + application_version: None, + parser_version: CODEX_ACCOUNT_EVIDENCE_PARSER_VERSION.to_string(), + artifact_kind: "auth_json".to_string(), + artifact_path_hash: artifact_path_hash.clone(), + record_fingerprint: record_fingerprint.clone(), + }); + + let plan_allowed = claims.auth_mode.as_deref().is_none_or(|mode| { + !matches!( + mode.trim().to_ascii_lowercase().as_str(), + "api" | "api_key" | "apikey" ) }); + if plan_allowed { + if let Some(raw_plan_name) = claims.plan_type { + let observation_id = account_plan_observation_id( + &source.source_id, + provider_account_id.as_ref(), + &raw_plan_name, + observed_at, + AccountEvidenceKind::AuthSnapshot, + ); + scan.plan_observations.push(AccountPlanObservationV1 { + schema_version: ACCOUNT_PLAN_OBSERVATION_SCHEMA_VERSION.to_string(), + observation_id, + provider: CODEX_PROVIDER.to_string(), + source_id: source.source_id.clone(), + provider_account_id, + plan_name: normalize_plan_name(&raw_plan_name), + raw_plan_name, + observed_at, + active_from: claims.active_from, + active_until: claims.active_until, + is_current_snapshot: true, + evidence_kind: AccountEvidenceKind::AuthSnapshot, + confidence: Confidence::High, + parser_version: CODEX_ACCOUNT_EVIDENCE_PARSER_VERSION.to_string(), + artifact_path_hash, + record_fingerprint, + }); + } + } +} + +fn collect_codex_telemetry_evidence( + source: &SourceLocation, + root: &Path, + checkpoints: &[AccountEvidenceCheckpointV1], + scan: &mut AccountEvidenceScan, +) -> Result<()> { + let account_attribute = Regex::new( + r#"(?i)(?:\"?user\.account_id\"?|\"?user_account_id\"?)\s*[:=]\s*[\"']?([a-z0-9_:\-]{3,256})"#, + )?; + let email_attribute = Regex::new( + r#"(?i)\"?user\.email\"?\s*[:=]\s*[\"']?([a-z0-9.!#$%&'*+/=?^_`{|}~\-]+@[a-z0-9.\-]+\.[a-z]{2,})"#, + )?; + let conversation_attribute = Regex::new( + r#"(?i)(?:\"?conversation\.id\"?|\"?conversation_id\"?|\"?conversationId\"?)\s*[:=]\s*[\"']?([a-z0-9_:\-]{3,256})"#, + )?; + let auth_mode_attribute = Regex::new( + r#"(?i)(?:\"?auth\.mode\"?|\"?auth_mode\"?)\s*[:=]\s*[\"']?([a-z0-9_:\-]{2,64})"#, + )?; + let app_version_attribute = Regex::new( + r#"(?i)(?:\"?app\.version\"?|\"?application_version\"?|\"?cli_version\"?)\s*[:=]\s*[\"']?([a-z0-9.+_\-]{1,64})"#, + )?; + let event_name = Regex::new(r#"(?:^|\s)event\.name\s*=\s*\"([a-z0-9._-]+)\""#)?; + let auth_reload = + Regex::new(r#"(?i)^\s*Reloading auth for account\s+([a-z0-9_:\-]{3,256})(?:\s|$)"#)?; + + for database_path in [ + root.join("logs_2.sqlite"), + root.join("sqlite/logs_2.sqlite"), + ] { + if !database_path.is_file() { + continue; + } + let Ok(connection) = open_sqlite_readonly(&database_path) else { + continue; + }; + let Ok(has_logs_table) = sqlite_table_exists(&connection, "logs") else { + continue; + }; + if !has_logs_table { + continue; + } + let Ok(maximum_row_id) = + connection.query_row("SELECT COALESCE(MAX(id), 0) FROM logs", [], |row| { + row.get::<_, i64>(0) + }) + else { + continue; + }; + let checkpoint_row_fingerprint = + codex_telemetry_checkpoint_row_fingerprint(&connection, maximum_row_id); + let current_state = + codex_telemetry_file_state(&database_path, maximum_row_id, checkpoint_row_fingerprint); + let path_hash = hash_text(&canonical_display(&database_path)); + let previous_state = checkpoints + .iter() + .find(|checkpoint| { + checkpoint.source_id == source.source_id + && checkpoint.artifact_path_hash == path_hash + && checkpoint.parser_version == CODEX_ACCOUNT_EVIDENCE_PARSER_VERSION + }) + .map(codex_telemetry_cursor); + if previous_state.as_ref() == Some(¤t_state) { + continue; + } + let database_generation_matches = previous_state.as_ref().is_some_and(|previous| { + previous + .checkpoint_row_fingerprint + .as_ref() + .is_some_and(|expected| { + codex_telemetry_checkpoint_row_fingerprint(&connection, previous.maximum_row_id) + .as_ref() + == Some(expected) + }) + }); + let minimum_row_id = previous_state + .as_ref() + .filter(|previous| { + database_generation_matches + && maximum_row_id > previous.maximum_row_id + && current_state.database_size >= previous.database_size + }) + .map_or(i64::MIN, |previous| previous.maximum_row_id); + let Ok(mut statement) = connection.prepare( + r#" + SELECT id, ts, ts_nanos, target, feedback_log_body + FROM logs + WHERE feedback_log_body IS NOT NULL + AND id > ?1 + AND ( + ( + target IN ('codex_otel.log_only', 'codex_otel.trace_safe') + AND instr(feedback_log_body, 'codex.conversation_starts') > 0 + AND ( + instr(feedback_log_body, 'user.account_id') > 0 + OR instr(feedback_log_body, 'user.email') > 0 + ) + ) + OR ( + target LIKE 'codex_core::auth%' + AND instr(feedback_log_body, 'Reloading auth for account') > 0 + ) + ) + ORDER BY id + "#, + ) else { + continue; + }; + let Ok(rows) = statement.query_map([minimum_row_id], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + )) + }) else { + continue; + }; + let mut read_complete = true; + for row in rows { + let Ok((row_id, seconds, nanos, target, body)) = row else { + read_complete = false; + break; + }; + let Some(observed_at) = Utc + .timestamp_opt(seconds, nanos.clamp(0, 999_999_999) as u32) + .single() + else { + continue; + }; + let structured_identity_event = matches!( + target.as_str(), + "codex_otel.log_only" | "codex_otel.trace_safe" + ) && event_name + .captures(&body) + .and_then(|captures| captures.get(1)) + .is_some_and(|value| value.as_str() == "codex.conversation_starts"); + let reload_account_id = target + .starts_with("codex_core::auth") + .then(|| { + auth_reload + .captures(&body) + .and_then(|captures| captures.get(1)) + .map(|value| value.as_str().to_string()) + }) + .flatten(); + let provider_user_id = reload_account_id.clone().or_else(|| { + structured_identity_event + .then(|| { + account_attribute + .captures(&body) + .and_then(|captures| captures.get(1)) + .map(|value| value.as_str().to_string()) + }) + .flatten() + }); + let email = structured_identity_event + .then(|| { + email_attribute + .captures(&body) + .and_then(|captures| captures.get(1)) + .map(|value| normalize_email(value.as_str())) + }) + .flatten(); + if provider_user_id.is_none() && email.is_none() { + continue; + } + let conversation_id = structured_identity_event + .then(|| { + conversation_attribute + .captures(&body) + .and_then(|captures| captures.get(1)) + .map(|value| value.as_str().to_string()) + }) + .flatten(); + let auth_mode = structured_identity_event + .then(|| { + auth_mode_attribute + .captures(&body) + .and_then(|captures| captures.get(1)) + .map(|value| value.as_str().to_string()) + }) + .flatten(); + let application_version = structured_identity_event + .then(|| { + app_version_attribute + .captures(&body) + .and_then(|captures| captures.get(1)) + .map(|value| value.as_str().to_string()) + }) + .flatten(); + let provider_account_id = provider_account_id_from_identity( + CODEX_PROVIDER, + provider_user_id.as_deref(), + email.as_deref(), + ); + let Some(provider_account_id) = provider_account_id else { + continue; + }; + let evidence_kind = if reload_account_id.is_some() { + AccountEvidenceKind::AuthReload + } else { + AccountEvidenceKind::TelemetryIdentity + }; + let conversation_id_hash = conversation_id.as_deref().map(hash_text); + let record_fingerprint = hash_text(&format!( + "codex-telemetry-identity.v1:{row_id}:{}:{}:{}:{}", + provider_user_id.as_deref().unwrap_or("none"), + email.as_deref().unwrap_or("none"), + conversation_id.as_deref().unwrap_or("none"), + observed_at.to_rfc3339() + )); + scan.accounts.push(ObservedProviderAccount { + provider_user_id: provider_user_id.clone(), + email: email.clone(), + plan_name: None, + observed_at, + }); + scan.identity_observations + .push(AccountIdentityObservationV1 { + schema_version: ACCOUNT_IDENTITY_OBSERVATION_SCHEMA_VERSION.to_string(), + observation_id: account_identity_observation_id( + &source.source_id, + evidence_kind, + observed_at, + &record_fingerprint, + ), + provider: CODEX_PROVIDER.to_string(), + source_id: source.source_id.clone(), + provider_account_id: Some(provider_account_id.clone()), + provider_user_id_hash: provider_user_id.as_deref().map(hash_text), + email_hash: email.as_deref().map(hash_text), + conversation_id_hash: conversation_id_hash.clone(), + turn_id_hash: None, + observed_at, + evidence_kind, + confidence: Confidence::High, + auth_mode, + application_version, + parser_version: CODEX_ACCOUNT_EVIDENCE_PARSER_VERSION.to_string(), + artifact_kind: "logs_2_sqlite".to_string(), + artifact_path_hash: path_hash.clone(), + record_fingerprint, + }); + if let Some(conversation_id_hash) = conversation_id_hash { + scan.conversation_bindings + .push(ConversationAccountBindingV1 { + schema_version: CONVERSATION_ACCOUNT_BINDING_SCHEMA_VERSION.to_string(), + binding_id: conversation_account_binding_id( + &source.source_id, + &conversation_id_hash, + None, + &provider_account_id, + ), + provider: CODEX_PROVIDER.to_string(), + source_id: source.source_id.clone(), + provider_account_id, + conversation_id_hash, + turn_id_hash: None, + observed_at, + evidence_kind, + confidence: Confidence::High, + }); + } + } + if read_complete { + scan.checkpoints.push(AccountEvidenceCheckpointV1 { + schema_version: ACCOUNT_EVIDENCE_CHECKPOINT_SCHEMA_VERSION.to_string(), + source_id: source.source_id.clone(), + artifact_path_hash: path_hash, + parser_version: CODEX_ACCOUNT_EVIDENCE_PARSER_VERSION.to_string(), + maximum_row_id: current_state.maximum_row_id, + checkpoint_row_fingerprint: current_state.checkpoint_row_fingerprint.clone(), + database_size: current_state.database_size, + database_modified_nanos: current_state.database_modified_nanos, + wal_size: current_state.wal_size, + wal_modified_nanos: current_state.wal_modified_nanos, + }); + } + } + Ok(()) +} + +fn collect_codex_reset_history_evidence( + source: &SourceLocation, + root: &Path, + scan: &mut AccountEvidenceScan, +) { + let state_path = root.join(".codex-global-state.json"); + let Some(value) = read_json_file(&state_path) else { + return; + }; + let Some(entries) = value + .pointer("/electron-persisted-atom-state/codex-rate-limit-reset-history") + .and_then(Value::as_array) + else { + return; + }; + let artifact_path_hash = hash_text(&canonical_display(&state_path)); + for entry in entries { + let Some(provider_user_id) = string_at_any(entry, &["accountId", "account_id"]) else { + continue; + }; + let Some(conversation_id) = string_at_any(entry, &["conversationId", "conversation_id"]) + else { + continue; + }; + let turn_id = string_at_any(entry, &["turnId", "turn_id"]); + let Some(observed_at) = entry + .get("occurredAtMs") + .or_else(|| entry.get("occurred_at_ms")) + .and_then(timestamp_from_scalar) + else { + continue; + }; + let Some(provider_account_id) = + provider_account_id_from_identity(CODEX_PROVIDER, Some(&provider_user_id), None) + else { + continue; + }; + let conversation_id_hash = hash_text(&conversation_id); + let turn_id_hash = turn_id.as_deref().map(hash_text); + let record_fingerprint = hash_text(&format!( + "codex-reset-history.v1:{provider_user_id}:{conversation_id}:{}:{}", + turn_id.as_deref().unwrap_or("none"), + observed_at.to_rfc3339() + )); + scan.accounts.push(ObservedProviderAccount { + provider_user_id: Some(provider_user_id.clone()), + email: None, + plan_name: None, + observed_at, + }); + scan.identity_observations + .push(AccountIdentityObservationV1 { + schema_version: ACCOUNT_IDENTITY_OBSERVATION_SCHEMA_VERSION.to_string(), + observation_id: account_identity_observation_id( + &source.source_id, + AccountEvidenceKind::ResetHistory, + observed_at, + &record_fingerprint, + ), + provider: CODEX_PROVIDER.to_string(), + source_id: source.source_id.clone(), + provider_account_id: Some(provider_account_id.clone()), + provider_user_id_hash: Some(hash_text(&provider_user_id)), + email_hash: None, + conversation_id_hash: Some(conversation_id_hash.clone()), + turn_id_hash: turn_id_hash.clone(), + observed_at, + evidence_kind: AccountEvidenceKind::ResetHistory, + confidence: Confidence::High, + auth_mode: None, + application_version: None, + parser_version: CODEX_ACCOUNT_EVIDENCE_PARSER_VERSION.to_string(), + artifact_kind: "global_state_reset_history".to_string(), + artifact_path_hash: artifact_path_hash.clone(), + record_fingerprint, + }); + scan.conversation_bindings + .push(ConversationAccountBindingV1 { + schema_version: CONVERSATION_ACCOUNT_BINDING_SCHEMA_VERSION.to_string(), + binding_id: conversation_account_binding_id( + &source.source_id, + &conversation_id_hash, + turn_id_hash.as_deref(), + &provider_account_id, + ), + provider: CODEX_PROVIDER.to_string(), + source_id: source.source_id.clone(), + provider_account_id, + conversation_id_hash, + turn_id_hash, + observed_at, + evidence_kind: AccountEvidenceKind::ResetHistory, + confidence: Confidence::High, + }); + } +} + +fn collect_codex_login_evidence( + source: &SourceLocation, + root: &Path, + scan: &mut AccountEvidenceScan, +) -> Result<()> { + let log_directory = root.join("log"); + let mut login_paths = std::fs::read_dir(&log_directory) + .ok() + .into_iter() + .flat_map(|entries| entries.filter_map(Result::ok)) + .filter_map(|entry| { + let name = entry.file_name(); + let name = name.to_str()?; + (name == "codex-login.log" || name.starts_with("codex-login.log.")) + .then(|| entry.path()) + }) + .collect::>(); + login_paths.sort(); + for login_path in login_paths { + let Ok(file) = File::open(&login_path) else { + continue; + }; + let artifact_path_hash = hash_text(&canonical_display(&login_path)); + let mut reader = BufReader::new(file); + let mut line = Vec::new(); + let mut line_number = 0u64; + loop { + match read_bounded_jsonl_line(&mut reader, &mut line, 64 * 1024)? { + BoundedLineRead::Eof => break, + BoundedLineRead::Oversized => { + line_number += 1; + continue; + } + BoundedLineRead::Complete => {} + } + line_number += 1; + let Ok(text) = std::str::from_utf8(&line) else { + continue; + }; + let lower = text.to_ascii_lowercase(); + if ![ + "successfully logged in", + "login successful", + "authentication successful", + "login completed", + ] + .iter() + .any(|marker| lower.contains(marker)) + { + continue; + } + let Some(timestamp) = text.split_whitespace().next() else { + continue; + }; + let Ok(observed_at) = DateTime::parse_from_rfc3339(timestamp) else { + continue; + }; + let observed_at = observed_at.with_timezone(&Utc); + let record_fingerprint = hash_text(&format!( + "codex-login-success.v1:{}:{line_number}:{}", + artifact_path_hash, + observed_at.to_rfc3339() + )); + scan.identity_observations + .push(AccountIdentityObservationV1 { + schema_version: ACCOUNT_IDENTITY_OBSERVATION_SCHEMA_VERSION.to_string(), + observation_id: account_identity_observation_id( + &source.source_id, + AccountEvidenceKind::LoginSuccess, + observed_at, + &record_fingerprint, + ), + provider: CODEX_PROVIDER.to_string(), + source_id: source.source_id.clone(), + provider_account_id: None, + provider_user_id_hash: None, + email_hash: None, + conversation_id_hash: None, + turn_id_hash: None, + observed_at, + evidence_kind: AccountEvidenceKind::LoginSuccess, + confidence: Confidence::Low, + auth_mode: None, + application_version: None, + parser_version: CODEX_ACCOUNT_EVIDENCE_PARSER_VERSION.to_string(), + artifact_kind: "codex_login_log".to_string(), + artifact_path_hash: artifact_path_hash.clone(), + record_fingerprint, + }); + } + } + Ok(()) +} + +fn dedupe_codex_account_evidence(scan: &mut AccountEvidenceScan) { + let mut account_keys = HashSet::new(); + scan.accounts.retain(|account| { + account_keys.insert((account.provider_user_id.clone(), account.email.clone())) + }); + let mut identity_ids = HashSet::new(); + scan.identity_observations + .retain(|observation| identity_ids.insert(observation.observation_id.clone())); + let mut plan_ids = HashSet::new(); + scan.plan_observations + .retain(|observation| plan_ids.insert(observation.observation_id.clone())); + let mut binding_ids = HashSet::new(); + scan.conversation_bindings + .retain(|binding| binding_ids.insert(binding.binding_id.clone())); +} +fn codex_auth_snapshot(root: &Path) -> Option { + let claims = codex_auth_claims(root)?; + let plan_name = claims.plan_type.as_deref().map(display_codex_plan_name); + let verified_at = claims.subscription_checked_at.or(claims.authenticated_at); Some(VerifiedSourceState { - provider_user_id, - email, + provider_user_id: claims.provider_user_id, + email: claims.email, account_label: None, plan_name, - authenticated_at, + authenticated_at: claims.authenticated_at, verified_at, - subscription, + // Provider plan detection is intentionally separate from user-entered billing facts. + subscription: None, }) } @@ -7773,32 +8619,6 @@ fn display_codex_plan_name(plan_type: &str) -> String { } } -fn codex_verified_subscription( - plan_type: &str, - paid_at: Option>, - current_period_ends_at: Option>, - verified_at: Option>, -) -> Option { - let started_at = paid_at?; - let (plan_name, price) = match plan_type.trim().to_ascii_lowercase().as_str() { - "plus" => ("Plus".to_string(), 2000), - "pro" => ("Pro".to_string(), 20000), - _ => return None, - }; - Some(VerifiedSubscriptionState { - plan_name, - price, - currency: "USD".to_string(), - billing_period: BillingPeriod::Monthly, - paid_at, - started_at, - ended_at: None, - current_period_ends_at, - status: SubscriptionStatus::Active, - verified_at, - }) -} - fn jwt_payload_value(token: &str) -> Option { let payload = token.split('.').nth(1)?; let bytes = decode_base64_url(payload)?; @@ -11969,20 +12789,21 @@ mod tests { verified.verified_at.map(|value| value.to_rfc3339()), Some("2026-05-29T10:14:56.058278+00:00".to_string()) ); - let subscription = verified.subscription.as_ref().expect("subscription"); - assert_eq!(subscription.plan_name, "Plus"); - assert_eq!(subscription.price, 2000); + assert!(verified.subscription.is_none()); + let evidence = CodexAdapter + .collect_account_evidence(&source, &[]) + .expect("account evidence"); + let plan = evidence.plan_observations.first().expect("plan evidence"); + assert_eq!(plan.raw_plan_name, "plus"); + assert_eq!(plan.plan_name, "Plus"); assert_eq!( - subscription.started_at.to_rfc3339(), + plan.active_from.expect("active from").to_rfc3339(), "2026-05-29T10:12:43+00:00" ); assert_eq!( - subscription - .current_period_ends_at - .map(|value| value.to_rfc3339()), + plan.active_until.map(|value| value.to_rfc3339()), Some("2026-06-29T10:12:43+00:00".to_string()) ); - assert_eq!(subscription.ended_at, None); assert_eq!(scan.events[0].provider_account_id, None); assert_ne!( scan.events[0] @@ -12042,20 +12863,20 @@ mod tests { verified.verified_at.map(|value| value.to_rfc3339()), Some("2026-05-29T10:14:56.058278+00:00".to_string()) ); - let subscription = verified.subscription.as_ref().expect("subscription"); - assert_eq!(subscription.plan_name, "Plus"); - assert_eq!(subscription.price, 2000); + assert!(verified.subscription.is_none()); + let evidence = CodexAdapter + .collect_account_evidence(&source, &[]) + .expect("account evidence"); + let plan = evidence.plan_observations.first().expect("plan evidence"); + assert_eq!(plan.raw_plan_name, "plus"); assert_eq!( - subscription.started_at.to_rfc3339(), + plan.active_from.expect("active from").to_rfc3339(), "2026-05-29T10:12:43+00:00" ); assert_eq!( - subscription - .current_period_ends_at - .map(|value| value.to_rfc3339()), + plan.active_until.map(|value| value.to_rfc3339()), Some("2026-06-29T10:12:43+00:00".to_string()) ); - assert_eq!(subscription.ended_at, None); assert_eq!(scan.events[0].provider_account_id, None); } @@ -12092,8 +12913,317 @@ mod tests { }; assert!(verified.verified_at.is_some()); - let subscription = verified.subscription.expect("subscription"); - assert_eq!(subscription.verified_at, None); + assert!(verified.subscription.is_none()); + let evidence = CodexAdapter + .collect_account_evidence(&source, &[]) + .expect("account evidence"); + assert_eq!(evidence.plan_observations.len(), 1); + assert!(evidence.plan_observations[0].is_current_snapshot); + assert_eq!( + evidence.plan_observations[0] + .active_until + .map(|value| value.to_rfc3339()), + Some("2026-06-29T10:12:43+00:00".to_string()) + ); + } + + #[test] + fn codex_collects_allowlisted_telemetry_reset_and_login_evidence() { + let dir = tempfile::tempdir().expect("tempdir"); + let database_path = dir.path().join("logs_2.sqlite"); + let connection = Connection::open(&database_path).expect("logs database"); + connection + .execute_batch( + r#" + CREATE TABLE logs ( + id INTEGER PRIMARY KEY, + ts INTEGER, + ts_nanos INTEGER, + target TEXT NOT NULL, + feedback_log_body TEXT + ); + "#, + ) + .expect("logs schema"); + connection + .execute( + "INSERT INTO logs (id, ts, ts_nanos, target, feedback_log_body) VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params![ + 1, + 1_787_227_200_i64, + 123_i64, + "codex_otel.log_only", + "event.name=\"codex.conversation_starts\" user.account_id=acct-telemetry user.email=owner@example.test conversation.id=conversation-1 auth.mode=chatgpt app.version=1.2.3" + ], + ) + .expect("telemetry row"); + connection + .execute( + "INSERT INTO logs (id, ts, ts_nanos, target, feedback_log_body) VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params![ + 2, + 1_787_227_201_i64, + 0_i64, + "codex_core::auth", + "Reloading auth for account acct-reloaded" + ], + ) + .expect("reload row"); + connection + .execute( + "INSERT INTO logs (id, ts, ts_nanos, target, feedback_log_body) VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params![ + 3, + 1_787_227_202_i64, + 0_i64, + "codex_otel.log_only", + "event.name=\"codex.user_prompt\" prompt=\"Please discuss user.account_id=acct-prompt user.email=prompt@example.test conversation.id=conversation-prompt\"" + ], + ) + .expect("arbitrary text row"); + drop(connection); + + std::fs::write( + dir.path().join(".codex-global-state.json"), + serde_json::json!({ + "electron-persisted-atom-state": { + "codex-rate-limit-reset-history": [{ + "accountId": "acct-reset", + "conversationId": "conversation-reset", + "turnId": "turn-reset", + "occurredAtMs": 1_787_227_203_000_i64 + }] + } + }) + .to_string(), + ) + .expect("global state"); + std::fs::create_dir_all(dir.path().join("log")).expect("log directory"); + std::fs::write( + dir.path().join("log/codex-login.log.1"), + "2026-08-20T12:00:04Z login successful for arbitrary@example.test acct-visible-only-in-body\n", + ) + .expect("rotated login log"); + std::fs::write( + dir.path().join("log/codex-login.log"), + "2026-08-20T12:00:05Z unrelated message\n2026-08-20T12:00:06Z login completed\n", + ) + .expect("login log"); + let source = SourceLocation::local_adapter( + CODEX_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + + let evidence = CodexAdapter + .collect_account_evidence(&source, &[]) + .expect("account evidence"); + + assert_eq!( + evidence + .identity_observations + .iter() + .filter(|item| item.evidence_kind == AccountEvidenceKind::TelemetryIdentity) + .count(), + 1 + ); + assert_eq!( + evidence + .identity_observations + .iter() + .filter(|item| item.evidence_kind == AccountEvidenceKind::AuthReload) + .count(), + 1 + ); + assert_eq!( + evidence + .identity_observations + .iter() + .filter(|item| item.evidence_kind == AccountEvidenceKind::ResetHistory) + .count(), + 1 + ); + let login_observations = evidence + .identity_observations + .iter() + .filter(|item| item.evidence_kind == AccountEvidenceKind::LoginSuccess) + .collect::>(); + assert_eq!(login_observations.len(), 2); + assert!(login_observations.iter().all(|item| { + item.provider_account_id.is_none() + && item.provider_user_id_hash.is_none() + && item.email_hash.is_none() + })); + assert_eq!(evidence.conversation_bindings.len(), 2); + assert!(evidence + .conversation_bindings + .iter() + .all(|binding| binding.conversation_id_hash.len() == 64)); + assert!(evidence + .conversation_bindings + .iter() + .all(|binding| binding.conversation_id_hash != "conversation-1")); + + let retry_before_ack = CodexAdapter + .collect_account_evidence(&source, &[]) + .expect("retry account evidence before acknowledgement"); + assert_eq!( + retry_before_ack + .identity_observations + .iter() + .filter(|item| matches!( + item.evidence_kind, + AccountEvidenceKind::TelemetryIdentity | AccountEvidenceKind::AuthReload + )) + .count(), + 2, + "telemetry must remain retryable until the caller commits the evidence" + ); + let committed_checkpoints = evidence.checkpoints.clone(); + + let repeated = CodexAdapter + .collect_account_evidence(&source, &committed_checkpoints) + .expect("repeat account evidence after checkpoint commit"); + assert_eq!( + repeated + .identity_observations + .iter() + .filter(|item| matches!( + item.evidence_kind, + AccountEvidenceKind::TelemetryIdentity | AccountEvidenceKind::AuthReload + )) + .count(), + 0, + "an unchanged telemetry database must not be rescanned" + ); + + let connection = Connection::open(&database_path).expect("reopen logs database"); + connection + .execute( + "INSERT INTO logs (id, ts, ts_nanos, target, feedback_log_body) VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params![ + 4, + 1_787_227_204_i64, + 0_i64, + "codex_otel.trace_safe", + "event.name=\"codex.conversation_starts\" user.account_id=acct-appended" + ], + ) + .expect("append telemetry row"); + drop(connection); + let appended = CodexAdapter + .collect_account_evidence(&source, &committed_checkpoints) + .expect("incremental account evidence"); + assert_eq!( + appended + .identity_observations + .iter() + .filter(|item| item.evidence_kind == AccountEvidenceKind::TelemetryIdentity) + .count(), + 1, + "only the appended telemetry range should be parsed" + ); + assert_eq!(appended.checkpoints.len(), 1); + + std::fs::rename(&database_path, dir.path().join("logs_2.previous.sqlite")) + .expect("archive replaced telemetry database"); + let mut replacement = Connection::open(&database_path).expect("replacement logs database"); + replacement + .execute_batch( + r#" + CREATE TABLE logs ( + id INTEGER PRIMARY KEY, + ts INTEGER, + ts_nanos INTEGER, + target TEXT NOT NULL, + feedback_log_body TEXT + ); + "#, + ) + .expect("replacement logs schema"); + let transaction = replacement.transaction().expect("replacement transaction"); + for row_id in 1..=100_i64 { + let body = match row_id { + 1 => "event.name=\"codex.conversation_starts\" user.account_id=acct-replacement-early".to_string(), + 100 => "event.name=\"codex.conversation_starts\" user.account_id=acct-replacement-late".to_string(), + _ => format!("replacement filler row {row_id} {}", "x".repeat(256)), + }; + transaction + .execute( + "INSERT INTO logs (id, ts, ts_nanos, target, feedback_log_body) VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params![ + row_id, + 1_787_227_300_i64 + row_id, + 0_i64, + "codex_otel.log_only", + body + ], + ) + .expect("replacement telemetry row"); + } + transaction.commit().expect("commit replacement telemetry"); + drop(replacement); + + let replacement_scan = CodexAdapter + .collect_account_evidence(&source, &appended.checkpoints) + .expect("replacement telemetry evidence"); + let replacement_accounts = replacement_scan + .accounts + .iter() + .filter_map(|account| account.provider_user_id.as_deref()) + .collect::>(); + assert!(replacement_accounts.contains("acct-replacement-early")); + assert!(replacement_accounts.contains("acct-replacement-late")); + assert_eq!(replacement_scan.checkpoints.len(), 1); + } + + #[test] + fn codex_skips_malformed_or_locked_telemetry_databases() { + let malformed = tempfile::tempdir().expect("malformed tempdir"); + let malformed_connection = + Connection::open(malformed.path().join("logs_2.sqlite")).expect("malformed database"); + malformed_connection + .execute_batch("CREATE TABLE logs (id INTEGER PRIMARY KEY, unexpected TEXT);") + .expect("malformed schema"); + drop(malformed_connection); + let malformed_source = SourceLocation::local_adapter( + CODEX_PROVIDER, + "test", + "0", + malformed.path(), + LocationOrigin::Configured, + ); + assert!(CodexAdapter + .collect_account_evidence(&malformed_source, &[]) + .expect("malformed database is non-fatal") + .identity_observations + .is_empty()); + + let locked = tempfile::tempdir().expect("locked tempdir"); + let locked_connection = + Connection::open(locked.path().join("logs_2.sqlite")).expect("locked database"); + locked_connection + .execute_batch( + "CREATE TABLE logs (id INTEGER PRIMARY KEY, ts INTEGER, ts_nanos INTEGER, feedback_log_body TEXT); PRAGMA locking_mode=EXCLUSIVE; BEGIN EXCLUSIVE;", + ) + .expect("exclusive lock"); + let locked_source = SourceLocation::local_adapter( + CODEX_PROVIDER, + "test", + "0", + locked.path(), + LocationOrigin::Configured, + ); + assert!(CodexAdapter + .collect_account_evidence(&locked_source, &[]) + .expect("locked database is non-fatal") + .identity_observations + .is_empty()); + locked_connection + .execute_batch("ROLLBACK") + .expect("unlock database"); } #[test] diff --git a/crates/statsai-core/src/account_plan.rs b/crates/statsai-core/src/account_plan.rs new file mode 100644 index 0000000..6a0c7d5 --- /dev/null +++ b/crates/statsai-core/src/account_plan.rs @@ -0,0 +1,315 @@ +//! Account identity and provider-plan evidence contracts. + +use crate::{hash_text, Confidence, ProviderAccountId, SourceId}; +use chrono::{DateTime, Utc}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +pub const ACCOUNT_IDENTITY_OBSERVATION_SCHEMA_VERSION: &str = "account_identity_observation.v1"; +pub const ACCOUNT_PLAN_OBSERVATION_SCHEMA_VERSION: &str = "account_plan_observation.v1"; +pub const CONVERSATION_ACCOUNT_BINDING_SCHEMA_VERSION: &str = "conversation_account_binding.v1"; +pub const ACCOUNT_PLAN_PROJECTION_SCHEMA_VERSION: &str = "account_plan_projection.v1"; +pub const ACCOUNT_EVIDENCE_SUMMARY_SCHEMA_VERSION: &str = "account_evidence_summary.v1"; + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum AccountEvidenceKind { + AuthSnapshot, + TelemetryIdentity, + AuthReload, + ResetHistory, + LoginSuccess, + QuotaStatus, + LegacyLocalAuth, +} + +impl AccountEvidenceKind { + #[must_use] + pub const fn is_strong_identity(self) -> bool { + matches!( + self, + Self::AuthSnapshot | Self::TelemetryIdentity | Self::AuthReload | Self::ResetHistory + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct AccountIdentityObservationV1 { + pub schema_version: String, + pub observation_id: String, + pub provider: String, + pub source_id: SourceId, + pub provider_account_id: Option, + pub provider_user_id_hash: Option, + pub email_hash: Option, + pub conversation_id_hash: Option, + pub turn_id_hash: Option, + pub observed_at: DateTime, + pub evidence_kind: AccountEvidenceKind, + pub confidence: Confidence, + pub auth_mode: Option, + pub application_version: Option, + pub parser_version: String, + pub artifact_kind: String, + pub artifact_path_hash: String, + pub record_fingerprint: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct AccountPlanObservationV1 { + pub schema_version: String, + pub observation_id: String, + pub provider: String, + pub source_id: SourceId, + pub provider_account_id: Option, + pub raw_plan_name: String, + pub plan_name: String, + pub observed_at: DateTime, + pub active_from: Option>, + pub active_until: Option>, + pub is_current_snapshot: bool, + pub evidence_kind: AccountEvidenceKind, + pub confidence: Confidence, + pub parser_version: String, + pub artifact_path_hash: String, + pub record_fingerprint: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct ConversationAccountBindingV1 { + pub schema_version: String, + pub binding_id: String, + pub provider: String, + pub source_id: SourceId, + pub provider_account_id: ProviderAccountId, + pub conversation_id_hash: String, + pub turn_id_hash: Option, + pub observed_at: DateTime, + pub evidence_kind: AccountEvidenceKind, + pub confidence: Confidence, +} + +pub const ACCOUNT_EVIDENCE_CHECKPOINT_SCHEMA_VERSION: &str = "account_evidence_checkpoint.v1"; + +/// Durable high-water mark for an incrementally scanned structured evidence artifact. +/// +/// `artifact_path_hash` intentionally keeps the local path out of the Store payload and sync +/// surface while still distinguishing multiple telemetry databases for one source. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct AccountEvidenceCheckpointV1 { + pub schema_version: String, + pub source_id: SourceId, + pub artifact_path_hash: String, + pub parser_version: String, + pub maximum_row_id: i64, + /// Fingerprint of the row at `maximum_row_id`, used to prove database continuity. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub checkpoint_row_fingerprint: Option, + pub database_size: i64, + pub database_modified_nanos: i64, + pub wal_size: i64, + pub wal_modified_nanos: i64, +} + +/// Privacy-safe device contribution sent to the hosted service. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct AccountPlanProjectionV1 { + pub schema_version: String, + pub projection_id: String, + pub semantic_fingerprint: String, + pub device_id: String, + pub provider: String, + pub provider_account_id: ProviderAccountId, + pub raw_plan_name: String, + pub plan_name: String, + pub observed_at: DateTime, + pub active_from: Option>, + pub active_until: Option>, + pub is_current_snapshot: bool, + pub evidence_kind: AccountEvidenceKind, + pub confidence: Confidence, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct AccountEvidenceSummaryV1 { + pub schema_version: String, + pub summary_id: String, + pub device_id: String, + pub provider: String, + pub provider_account_id: ProviderAccountId, + pub first_strong_observed_at: Option>, + pub last_strong_observed_at: Option>, + pub strong_observation_count: u64, + pub directly_bound_conversations: u64, + pub uncovered_gap_count: u64, + pub conflict_count: u64, + pub evidence_kinds: Vec, +} + +#[must_use] +pub fn normalize_plan_name(raw: &str) -> String { + raw.trim() + .split(['_', '-', ' ']) + .filter(|part| !part.is_empty()) + .map(|part| { + let mut chars = part.chars(); + chars.next().map_or_else(String::new, |first| { + format!( + "{}{}", + first.to_ascii_uppercase(), + chars.as_str().to_ascii_lowercase() + ) + }) + }) + .collect::>() + .join(" ") +} + +#[must_use] +pub fn account_identity_observation_id( + source_id: &SourceId, + kind: AccountEvidenceKind, + observed_at: DateTime, + record_fingerprint: &str, +) -> String { + format!( + "identity_observation_{}", + &hash_text(&format!( + "account_identity_observation.v1:{}:{kind:?}:{}:{record_fingerprint}", + source_id.0, + observed_at.to_rfc3339() + ))[..32] + ) +} + +#[must_use] +pub fn account_plan_observation_id( + source_id: &SourceId, + provider_account_id: Option<&ProviderAccountId>, + raw_plan_name: &str, + observed_at: DateTime, + kind: AccountEvidenceKind, +) -> String { + format!( + "plan_observation_{}", + &hash_text(&format!( + "account_plan_observation.v1:{}:{}:{}:{}:{kind:?}", + source_id.0, + provider_account_id.map_or("unattributed", |id| id.0.as_str()), + raw_plan_name.trim().to_ascii_lowercase(), + observed_at.to_rfc3339() + ))[..32] + ) +} + +#[must_use] +pub fn conversation_account_binding_id( + source_id: &SourceId, + conversation_id_hash: &str, + turn_id_hash: Option<&str>, + provider_account_id: &ProviderAccountId, +) -> String { + format!( + "conversation_binding_{}", + &hash_text(&format!( + "conversation_account_binding.v1:{}:{conversation_id_hash}:{}:{}", + source_id.0, + turn_id_hash.unwrap_or("none"), + provider_account_id.0 + ))[..32] + ) +} + +#[must_use] +pub fn plan_projection_from_observation( + observation: &AccountPlanObservationV1, + device_id: &str, +) -> Option { + let provider_account_id = observation.provider_account_id.clone()?; + let semantic_fingerprint = hash_text(&format!( + "account_plan_projection.v1:{}:{}:{}:{}:{}:{}:{}:{:?}", + observation.provider, + provider_account_id.0, + observation.raw_plan_name.trim().to_ascii_lowercase(), + observation.observed_at.to_rfc3339(), + observation + .active_from + .map_or_else(|| "none".to_string(), |value| value.to_rfc3339()), + observation + .active_until + .map_or_else(|| "none".to_string(), |value| value.to_rfc3339()), + observation.is_current_snapshot, + observation.evidence_kind + )); + let projection_id = format!( + "account_plan_projection_{}", + &hash_text(&format!("{device_id}:{semantic_fingerprint}"))[..32] + ); + Some(AccountPlanProjectionV1 { + schema_version: ACCOUNT_PLAN_PROJECTION_SCHEMA_VERSION.to_string(), + projection_id, + semantic_fingerprint, + device_id: device_id.to_string(), + provider: observation.provider.clone(), + provider_account_id, + raw_plan_name: observation.raw_plan_name.clone(), + plan_name: observation.plan_name.clone(), + observed_at: observation.observed_at, + active_from: observation.active_from, + active_until: observation.active_until, + is_current_snapshot: observation.is_current_snapshot, + evidence_kind: observation.evidence_kind, + confidence: observation.confidence.clone(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone; + + #[test] + fn unknown_plan_names_remain_valid_and_readable() { + assert_eq!( + normalize_plan_name("future_ultra-enterprise"), + "Future Ultra Enterprise" + ); + } + + #[test] + fn plan_observation_ids_are_deterministic_and_account_scoped() { + let source_id = SourceId("source".to_string()); + let observed_at = Utc + .with_ymd_and_hms(2026, 8, 23, 0, 0, 0) + .single() + .expect("timestamp"); + let account_a = ProviderAccountId("account-a".to_string()); + let account_b = ProviderAccountId("account-b".to_string()); + let first = account_plan_observation_id( + &source_id, + Some(&account_a), + "future_ultra", + observed_at, + AccountEvidenceKind::AuthSnapshot, + ); + let repeat = account_plan_observation_id( + &source_id, + Some(&account_a), + "future_ultra", + observed_at, + AccountEvidenceKind::AuthSnapshot, + ); + let other_account = account_plan_observation_id( + &source_id, + Some(&account_b), + "future_ultra", + observed_at, + AccountEvidenceKind::AuthSnapshot, + ); + + assert_eq!(first, repeat); + assert_ne!(first, other_account); + } +} diff --git a/crates/statsai-core/src/lib.rs b/crates/statsai-core/src/lib.rs index 3ff7cba..c06f114 100644 --- a/crates/statsai-core/src/lib.rs +++ b/crates/statsai-core/src/lib.rs @@ -1,5 +1,6 @@ //! Core schemas and ID helpers for `statsai`. +mod account_plan; mod archive; mod code_changes; mod quota; @@ -11,6 +12,7 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::path::{Path, PathBuf}; +pub use account_plan::*; pub use archive::*; pub use code_changes::*; pub use quota::*; @@ -28,12 +30,14 @@ pub const SYNC_BATCH_V1_SCHEMA_VERSION: &str = "sync_batch.v1"; pub const SYNC_BATCH_V2_SCHEMA_VERSION: &str = "sync_batch.v2"; pub const SYNC_BATCH_V3_SCHEMA_VERSION: &str = "sync_batch.v3"; pub const SYNC_BATCH_V4_SCHEMA_VERSION: &str = "sync_batch.v4"; +pub const SYNC_BATCH_V5_SCHEMA_VERSION: &str = "sync_batch.v5"; pub const SYNC_ACK_V1_SCHEMA_VERSION: &str = "sync_ack.v1"; pub const SYNC_ACK_V2_SCHEMA_VERSION: &str = "sync_ack.v2"; pub const SYNC_ACK_V3_SCHEMA_VERSION: &str = "sync_ack.v3"; pub const SYNC_ACK_V4_SCHEMA_VERSION: &str = "sync_ack.v4"; -pub const SYNC_BATCH_SCHEMA_VERSION: &str = SYNC_BATCH_V4_SCHEMA_VERSION; -pub const SYNC_ACK_SCHEMA_VERSION: &str = SYNC_ACK_V4_SCHEMA_VERSION; +pub const SYNC_ACK_V5_SCHEMA_VERSION: &str = "sync_ack.v5"; +pub const SYNC_BATCH_SCHEMA_VERSION: &str = SYNC_BATCH_V5_SCHEMA_VERSION; +pub const SYNC_ACK_SCHEMA_VERSION: &str = SYNC_ACK_V5_SCHEMA_VERSION; #[derive( Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema, @@ -733,6 +737,15 @@ pub struct SyncBatch { /// plans, credits, and sample counts are deliberately absent from this type. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub quota_cycle_contributions: Vec, + /// Plan labels carrying only the canonical account reference, provider bounds, + /// and evidence grade. Emails, provider user IDs, conversation and turn IDs, + /// artifact paths, and raw provenance are deliberately absent from this type. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub account_plan_observations: Vec, + /// Aggregate coverage and conflict counts describing how well each account is + /// evidenced. Individual observations never leave the device through this type. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub account_evidence_summaries: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub authoritative_snapshot: Option, pub created_at: DateTime, @@ -757,6 +770,10 @@ pub struct SyncAuthoritativeSnapshot { pub code_change_metric_ids: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub quota_cycle_contribution_ids: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub account_plan_observation_ids: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub account_evidence_summary_ids: Vec, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -806,6 +823,10 @@ pub struct SyncEntityCounts { pub code_change_metrics: u64, #[serde(default, skip_serializing_if = "sync_count_is_zero")] pub quota_cycle_contributions: u64, + #[serde(default, skip_serializing_if = "sync_count_is_zero")] + pub account_plan_observations: u64, + #[serde(default, skip_serializing_if = "sync_count_is_zero")] + pub account_evidence_summaries: u64, } fn sync_count_is_zero(value: &u64) -> bool { @@ -2149,6 +2170,8 @@ mod tests { task_verifications: 0, code_change_metrics: 0, quota_cycle_contributions: 0, + account_plan_observations: 0, + account_evidence_summaries: 0, }, duplicates: SyncEntityCounts { sources: 0, @@ -2161,6 +2184,8 @@ mod tests { task_verifications: 0, code_change_metrics: 0, quota_cycle_contributions: 0, + account_plan_observations: 0, + account_evidence_summaries: 0, }, rejected: Vec::new(), }; @@ -2191,6 +2216,8 @@ mod tests { task_verifications: 1, code_change_metrics: 2, quota_cycle_contributions: 0, + account_plan_observations: 0, + account_evidence_summaries: 0, }, duplicates: SyncEntityCounts { sources: 0, @@ -2203,6 +2230,8 @@ mod tests { task_verifications: 0, code_change_metrics: 0, quota_cycle_contributions: 0, + account_plan_observations: 0, + account_evidence_summaries: 0, }, rejected: Vec::new(), }; @@ -2213,6 +2242,8 @@ mod tests { assert_eq!(json["accepted"]["task_verifications"], 1); assert_eq!(json["accepted"]["code_change_metrics"], 2); assert!(json["accepted"].get("quota_cycle_contributions").is_none()); + assert!(json["accepted"].get("account_plan_observations").is_none()); + assert!(json["accepted"].get("account_evidence_summaries").is_none()); } #[test] @@ -2231,6 +2262,8 @@ mod tests { task_verifications: 0, code_change_metrics: 1, quota_cycle_contributions: 4, + account_plan_observations: 0, + account_evidence_summaries: 0, }, duplicates: SyncEntityCounts { sources: 0, @@ -2243,6 +2276,8 @@ mod tests { task_verifications: 0, code_change_metrics: 0, quota_cycle_contributions: 0, + account_plan_observations: 0, + account_evidence_summaries: 0, }, rejected: Vec::new(), }; diff --git a/crates/statsai-daemon/src/lib.rs b/crates/statsai-daemon/src/lib.rs index 6e30c7d..34e5f03 100644 --- a/crates/statsai-daemon/src/lib.rs +++ b/crates/statsai-daemon/src/lib.rs @@ -5,8 +5,8 @@ use serde_json::json; use statsai_core::{ SyncAck, SyncBatch, SyncEntityCounts, SyncRejectedRecord, SYNC_ACK_V1_SCHEMA_VERSION, SYNC_ACK_V2_SCHEMA_VERSION, SYNC_ACK_V3_SCHEMA_VERSION, SYNC_ACK_V4_SCHEMA_VERSION, - SYNC_BATCH_V1_SCHEMA_VERSION, SYNC_BATCH_V2_SCHEMA_VERSION, SYNC_BATCH_V3_SCHEMA_VERSION, - SYNC_BATCH_V4_SCHEMA_VERSION, + SYNC_ACK_V5_SCHEMA_VERSION, SYNC_BATCH_V1_SCHEMA_VERSION, SYNC_BATCH_V2_SCHEMA_VERSION, + SYNC_BATCH_V3_SCHEMA_VERSION, SYNC_BATCH_V4_SCHEMA_VERSION, SYNC_BATCH_V5_SCHEMA_VERSION, }; use statsai_store::Store; use std::io::Read; @@ -32,6 +32,7 @@ fn sync_ack_schema_version(batch_schema_version: &str) -> Result<&'static str> { SYNC_BATCH_V2_SCHEMA_VERSION => Ok(SYNC_ACK_V2_SCHEMA_VERSION), SYNC_BATCH_V3_SCHEMA_VERSION => Ok(SYNC_ACK_V3_SCHEMA_VERSION), SYNC_BATCH_V4_SCHEMA_VERSION => Ok(SYNC_ACK_V4_SCHEMA_VERSION), + SYNC_BATCH_V5_SCHEMA_VERSION => Ok(SYNC_ACK_V5_SCHEMA_VERSION), other => bail!("unsupported sync batch schema {other}"), } } @@ -222,21 +223,30 @@ pub fn ingest_sync_batch(store: &Store, batch: &SyncBatch) -> Result { && batch.schema_version != SYNC_BATCH_V2_SCHEMA_VERSION && batch.schema_version != SYNC_BATCH_V3_SCHEMA_VERSION && batch.schema_version != SYNC_BATCH_V4_SCHEMA_VERSION + && batch.schema_version != SYNC_BATCH_V5_SCHEMA_VERSION { bail!("unsupported sync batch schema {}", batch.schema_version); } if !matches!( batch.schema_version.as_str(), - SYNC_BATCH_V3_SCHEMA_VERSION | SYNC_BATCH_V4_SCHEMA_VERSION + SYNC_BATCH_V3_SCHEMA_VERSION | SYNC_BATCH_V4_SCHEMA_VERSION | SYNC_BATCH_V5_SCHEMA_VERSION ) && !batch.code_change_metrics.is_empty() { bail!("code-change metrics require sync_batch.v3"); } - if batch.schema_version != SYNC_BATCH_V4_SCHEMA_VERSION - && !batch.quota_cycle_contributions.is_empty() + if !matches!( + batch.schema_version.as_str(), + SYNC_BATCH_V4_SCHEMA_VERSION | SYNC_BATCH_V5_SCHEMA_VERSION + ) && !batch.quota_cycle_contributions.is_empty() { bail!("quota cycle contributions require sync_batch.v4"); } + if batch.schema_version != SYNC_BATCH_V5_SCHEMA_VERSION + && (!batch.account_plan_observations.is_empty() + || !batch.account_evidence_summaries.is_empty()) + { + bail!("account-plan evidence requires sync_batch.v5"); + } if batch .code_change_metrics .iter() @@ -247,16 +257,19 @@ pub fn ingest_sync_batch(store: &Store, batch: &SyncBatch) -> Result { if batch.authoritative_snapshot.is_some() { bail!("authoritative snapshots are not supported by the loopback daemon"); } - // A local store holds quota observations and derives its own cycles from - // them; there is no table for another device's contributions. Acknowledging - // them as accepted told the sender to record them synced against this - // target and stop offering them, certifying a write that never happened. - // Refusing follows the authoritative-snapshot precedent above: this - // endpoint is for loopback diagnostics, and `/api/sync/batches` is the - // contract that stores quota cycles. + // A local store holds quota observations and account-plan evidence it + // derives from its own scans; there is no table for another device's + // contributions. Acknowledging them as accepted told the sender to record + // them synced against this target and stop offering them, certifying a + // write that never happened. Refusing follows the authoritative-snapshot + // precedent above: this endpoint is for loopback diagnostics, and + // `/api/sync/batches` is the contract that stores these collections. if !batch.quota_cycle_contributions.is_empty() { bail!("quota cycle contributions are not supported by the loopback daemon"); } + if !batch.account_plan_observations.is_empty() || !batch.account_evidence_summaries.is_empty() { + bail!("account-plan evidence is not supported by the loopback daemon"); + } let result = store.ingest_sync_batch(batch)?; @@ -274,6 +287,8 @@ pub fn ingest_sync_batch(store: &Store, batch: &SyncBatch) -> Result { task_verifications: result.merged_task_verifications, code_change_metrics: batch.code_change_metrics.len() as u64, quota_cycle_contributions: batch.quota_cycle_contributions.len() as u64, + account_plan_observations: batch.account_plan_observations.len() as u64, + account_evidence_summaries: batch.account_evidence_summaries.len() as u64, }, duplicates: SyncEntityCounts { sources: 0, @@ -287,6 +302,8 @@ pub fn ingest_sync_batch(store: &Store, batch: &SyncBatch) -> Result { .saturating_sub(result.merged_task_verifications), code_change_metrics: 0, quota_cycle_contributions: 0, + account_plan_observations: 0, + account_evidence_summaries: 0, }, rejected: Vec::::new(), }) @@ -321,16 +338,19 @@ mod watch { use chrono::{DateTime, Utc}; use notify::{Event, EventKind, RecursiveMode, Watcher}; use statsai_adapters::{ - adapter_for_provider, default_adapters, ProviderAdapter, ScanCandidateFile, ScanOptions, - VerifiedSourceObservation, + adapter_for_provider, default_adapters, remap_account_evidence_account_ids, + retain_accounts_referenced_by_account_evidence, AccountEvidenceScan, ProviderAdapter, + ScanCandidateFile, ScanOptions, VerifiedSourceObservation, }; use statsai_core::{ - hash_text, timestamp_in_period, IdentitySource, ProviderAccountId, SourceAccountAssignment, - SourceId, SourceKind, SourceLocation, SourceVerificationMode, UsageEvent, UsageSummary, + hash_text, provider_account_id_from_identity, timestamp_in_period, IdentitySource, + ProviderAccountId, SourceAccountAssignment, SourceId, SourceKind, SourceLocation, + SourceVerificationMode, UsageEvent, UsageSummary, }; use statsai_store::{ - reconcile_verified_source_state, verified_source_observation_hash, ScanFileReplacement, - ScanFileStateEntry, Store, + find_existing_provider_account, reconcile_verified_source_state, upsert_provider_account, + verified_source_observation_hash, ScanFileReplacement, ScanFileStateEntry, Store, + UpsertProviderAccountInput, }; use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; @@ -1003,6 +1023,51 @@ mod watch { verification_dependencies, ); for mut source in sources { + let verification_mode = source.verification_mode.clone(); + let account_evidence_enabled = + matches!(verification_mode, SourceVerificationMode::Auto); + let mut account_evidence = statsai_adapters::AccountEvidenceScan::default(); + if account_evidence_enabled { + let account_evidence_checkpoints = scan_store + .account_evidence_checkpoints(&source.source_id) + .context("load account evidence checkpoints")?; + account_evidence = match adapter + .collect_account_evidence(&source, &account_evidence_checkpoints) + { + Ok(evidence) => evidence, + Err(error) => { + eprintln!( + "daemon: account evidence scan failed for {}: {error}", + source.path_label.as_deref().unwrap_or("unknown") + ); + failed = true; + continue; + } + }; + let known_account_aliases = canonicalize_known_account_evidence( + scan_store, + adapter.provider(), + &mut account_evidence, + ) + .context("canonicalize known account evidence")?; + scan_store + .retain_unseen_account_evidence( + &source.source_id, + &mut account_evidence.identity_observations, + &mut account_evidence.plan_observations, + &mut account_evidence.conversation_bindings, + ) + .context("filter previously collected account evidence")?; + retain_accounts_referenced_by_account_evidence( + adapter.provider(), + &known_account_aliases, + &mut account_evidence, + ); + } + let has_account_evidence = !account_evidence.identity_observations.is_empty() + || !account_evidence.plan_observations.is_empty() + || !account_evidence.conversation_bindings.is_empty() + || !account_evidence.checkpoints.is_empty(); let expected_data_version = commit_store .map(|_| scan_store.data_version()) .transpose() @@ -1061,7 +1126,6 @@ mod watch { .filter(|entry| !current_cache_keys.contains(entry.cache_key.as_str())) .collect::>(); let has_cache_entry_upgrades = !compatible_entries_to_upgrade.is_empty(); - let verification_mode = source.verification_mode.clone(); let probed_verified_source_state = if matches!(verification_mode, SourceVerificationMode::Disabled) { VerifiedSourceObservation::Unavailable @@ -1111,6 +1175,7 @@ mod watch { && removed_file_entries.is_empty() && !has_cache_entry_upgrades && !verified_state_changed + && !has_account_evidence { continue; } @@ -1199,6 +1264,30 @@ mod watch { store .upsert_source(source) .context("update source verified auth state")?; + if account_evidence_enabled { + canonicalize_account_evidence( + store, + adapter.provider(), + &mut account_evidence, + )?; + store.upsert_account_identity_observations( + &account_evidence.identity_observations, + )?; + store.upsert_account_plan_observations( + &account_evidence.plan_observations, + )?; + store.reconcile_source_account_evidence_assignments( + &source.source_id, + )?; + store.upsert_conversation_account_bindings( + &account_evidence.conversation_bindings, + )?; + store + .reattribute_conversation_bound_events(&source.source_id)?; + store.upsert_account_evidence_checkpoints( + &account_evidence.checkpoints, + )?; + } if pending_file_entries.is_empty() && removed_file_entries.is_empty() { @@ -1217,6 +1306,12 @@ mod watch { &mut scan.summaries, ) .context("resolve source accounts")?; + if account_evidence_enabled { + store.apply_conversation_account_bindings( + &source.source_id, + &mut scan.events, + )?; + } let replacement = store .replace_scan_file_records(ScanFileReplacement { source_id: &source.source_id, @@ -1397,6 +1492,65 @@ mod watch { Ok(()) } + fn canonicalize_account_evidence( + store: &Store, + provider: &str, + evidence: &mut AccountEvidenceScan, + ) -> Result<()> { + let mut canonical_ids = HashMap::new(); + for observed in &evidence.accounts { + let Some(detected_id) = provider_account_id_from_identity( + provider, + observed.provider_user_id.as_deref(), + observed.email.as_deref(), + ) else { + continue; + }; + let account = upsert_provider_account( + store, + UpsertProviderAccountInput { + provider, + provider_user_id: observed.provider_user_id.as_deref(), + email: observed.email.as_deref(), + label: None, + plan_name: None, + identity_source: Some(IdentitySource::LocalAuth), + verified_at: Some(observed.observed_at), + }, + )?; + canonical_ids.insert(detected_id, account.provider_account_id); + } + remap_account_evidence_account_ids(evidence, &canonical_ids); + Ok(()) + } + + fn canonicalize_known_account_evidence( + store: &Store, + provider: &str, + evidence: &mut AccountEvidenceScan, + ) -> Result> { + let mut canonical_ids = HashMap::new(); + for observed in &evidence.accounts { + let Some(detected_id) = provider_account_id_from_identity( + provider, + observed.provider_user_id.as_deref(), + observed.email.as_deref(), + ) else { + continue; + }; + if let Some(account) = find_existing_provider_account( + store, + provider, + observed.provider_user_id.as_deref(), + observed.email.as_deref(), + )? { + canonical_ids.insert(detected_id, account.provider_account_id); + } + } + remap_account_evidence_account_ids(evidence, &canonical_ids); + Ok(canonical_ids) + } + fn apply_account_resolution_to_event( assignments: &[SourceAccountAssignment], event: &mut UsageEvent, @@ -1951,6 +2105,49 @@ mod watch { } } + struct AccountEvidenceTrackingAdapter { + collect_calls: Arc>, + } + + impl ProviderAdapter for AccountEvidenceTrackingAdapter { + fn id(&self) -> &'static str { + "test-account-evidence" + } + + fn version(&self) -> &'static str { + "0" + } + + fn provider(&self) -> &'static str { + "codex" + } + + fn discover(&self) -> Vec { + Vec::new() + } + + fn scan_candidates(&self, _source: &SourceLocation) -> Result> { + Ok(Vec::new()) + } + + fn collect_account_evidence( + &self, + _source: &SourceLocation, + _checkpoints: &[statsai_core::AccountEvidenceCheckpointV1], + ) -> Result { + *self.collect_calls.lock().expect("collect calls") += 1; + Ok(statsai_adapters::AccountEvidenceScan::default()) + } + + fn scan( + &self, + _source: &SourceLocation, + _options: &ScanOptions, + ) -> Result { + Ok(statsai_adapters::AdapterScan::default()) + } + } + struct DuplicateFileAdapter { candidate: ScanCandidateFile, event: UsageEvent, @@ -2300,7 +2497,10 @@ mod watch { assert_eq!(*scan_calls.lock().expect("scan calls"), 0); assert_eq!(store.list_accounts().expect("accounts").len(), 1); - assert_eq!(store.list_subscriptions().expect("subscriptions").len(), 1); + assert!(store + .list_subscriptions() + .expect("subscriptions") + .is_empty()); let assignments = store .list_source_account_assignments_for_source(&source.source_id) .expect("assignments"); @@ -2316,7 +2516,7 @@ mod watch { // A watcher can observe auth.json while it is being rewritten. That // transiently produces no local snapshot, which must not end the - // account assignment or its verified subscription. + // account assignment. let unavailable_adapters: Vec> = vec![Box::new(TestAdapter { provider: "codex", verified_observation: VerifiedSourceObservation::Unavailable, @@ -2339,9 +2539,10 @@ mod watch { .expect("assignments after unavailable auth"); assert_eq!(assignments.len(), 1); assert_eq!(assignments[0].ended_at, None); - let subscriptions = store.list_subscriptions().expect("subscriptions"); - assert_eq!(subscriptions.len(), 1); - assert_eq!(subscriptions[0].ended_at, None); + assert!(store + .list_subscriptions() + .expect("subscriptions") + .is_empty()); let blocked_adapters: Vec> = vec![Box::new(TestAdapter { provider: "codex", @@ -2367,13 +2568,44 @@ mod watch { .expect("assignments after blocked auth"); assert_eq!(assignments.len(), 1); assert_eq!(assignments[0].ended_at, Some(blocked_since)); - let subscriptions = store.list_subscriptions().expect("subscriptions"); - assert_eq!(subscriptions.len(), 1); - assert_eq!(subscriptions[0].ended_at, None); + assert!(store + .list_subscriptions() + .expect("subscriptions") + .is_empty()); let _ = std::fs::remove_dir_all(&root); } + #[test] + fn manual_only_watcher_rescan_does_not_collect_account_evidence() { + let store = Store::in_memory().expect("store"); + let root = tempfile::tempdir().expect("source root"); + let mut source = SourceLocation::local_adapter( + "codex", + "test-account-evidence", + "0", + root.path(), + LocationOrigin::Configured, + ); + source.verification_mode = SourceVerificationMode::ManualOnly; + store.upsert_source(&source).expect("source"); + let collect_calls = Arc::new(Mutex::new(0)); + let adapters: Vec> = + vec![Box::new(AccountEvidenceTrackingAdapter { + collect_calls: Arc::clone(&collect_calls), + })]; + + rescan_changed_sources_with_adapters( + &store, + "device-test", + &[root.path().join("auth.json")], + &adapters, + ) + .expect("watcher rescan"); + + assert_eq!(*collect_calls.lock().expect("collect calls"), 0); + } + #[test] fn rescan_changed_sources_removes_records_for_deleted_files() { let store = Store::in_memory().expect("store"); @@ -2693,13 +2925,15 @@ mod tests { fn empty_batch() -> SyncBatch { SyncBatch { - schema_version: SYNC_BATCH_V3_SCHEMA_VERSION.to_string(), + schema_version: SYNC_BATCH_V4_SCHEMA_VERSION.to_string(), batch_id: "batch_test".to_string(), device_id: "device_test".to_string(), sources: Vec::new(), accounts: Vec::new(), source_account_assignments: Vec::new(), subscriptions: Vec::new(), + account_plan_observations: Vec::new(), + account_evidence_summaries: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: Vec::new(), @@ -2822,13 +3056,55 @@ mod tests { let store = Store::in_memory().expect("store"); let ack = ingest_sync_batch(&store, &empty_batch()).expect("ack"); - assert_eq!(ack.schema_version, SYNC_ACK_V3_SCHEMA_VERSION); + assert_eq!(ack.schema_version, SYNC_ACK_V4_SCHEMA_VERSION); assert_eq!(ack.batch_id, "batch_test"); assert_eq!(ack.accepted.events, 0); assert_eq!(ack.duplicates.events, 0); assert!(ack.rejected.is_empty()); } + #[test] + fn ingest_rejects_projection_collections_the_loopback_store_cannot_persist() { + let store = Store::in_memory().expect("store"); + let mut batch = empty_batch(); + batch.account_plan_observations.push( + serde_json::from_value(json!({ + "schema_version": "account_plan_projection.v1", + "projection_id": "projection-1", + "semantic_fingerprint": "fingerprint-1", + "device_id": batch.device_id, + "provider": "codex", + "provider_account_id": "account-1", + "raw_plan_name": "pro", + "plan_name": "Pro", + "observed_at": "2026-08-20T12:00:00Z", + "active_from": null, + "active_until": null, + "is_current_snapshot": true, + "evidence_kind": "auth_snapshot", + "confidence": "high" + })) + .expect("plan projection"), + ); + + // `empty_batch` is v4, which has no acknowledgement counter for these, + // so the version guard fires before the persistence one. + let versioned = ingest_sync_batch(&store, &batch).expect_err("account plan below v5"); + assert!(versioned + .to_string() + .contains("account-plan evidence requires sync_batch.v5")); + + batch.schema_version = SYNC_BATCH_V5_SCHEMA_VERSION.to_string(); + let error = ingest_sync_batch(&store, &batch).expect_err("unsupported projection"); + assert!(error + .to_string() + .contains("account-plan evidence is not supported by the loopback daemon")); + assert!(store + .account_plan_observations() + .expect("local plan ledger") + .is_empty()); + } + #[test] fn ingest_v1_batch_returns_v1_ack_schema() { let store = Store::in_memory().expect("store"); diff --git a/crates/statsai-store/src/account_plan.rs b/crates/statsai-store/src/account_plan.rs new file mode 100644 index 0000000..d9806d3 --- /dev/null +++ b/crates/statsai-store/src/account_plan.rs @@ -0,0 +1,1197 @@ +use super::{ + assignment_for_timestamp, is_verified_source_assignment, reattribute_source_records, + validate_time_window, Store, +}; +use anyhow::Result; +use rusqlite::params; +use statsai_core::{ + account_plan_observation_id, conversation_account_binding_id, hash_text, normalize_plan_name, + periods_overlap, plan_projection_from_observation, source_account_assignment_id, + AccountEvidenceCheckpointV1, AccountEvidenceKind, AccountEvidenceSummaryV1, + AccountIdentityObservationV1, AccountPlanObservationV1, AccountPlanProjectionV1, Confidence, + ConversationAccountBindingV1, IdentitySource, ProviderAccountId, QuotaObservationRecordV1, + SourceAccountAssignment, SourceId, UsageEvent, ACCOUNT_EVIDENCE_SUMMARY_SCHEMA_VERSION, + ACCOUNT_PLAN_OBSERVATION_SCHEMA_VERSION, SOURCE_ACCOUNT_ASSIGNMENT_SCHEMA_VERSION, +}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct AccountEvidenceReferenceCounts { + pub identity_observations: usize, + pub plan_observations: usize, + pub conversation_bindings: usize, +} + +impl AccountEvidenceReferenceCounts { + #[must_use] + pub const fn total(self) -> usize { + self.identity_observations + self.plan_observations + self.conversation_bindings + } +} + +impl Store { + pub fn account_evidence_checkpoints( + &self, + source_id: &SourceId, + ) -> Result> { + let mut statement = self.conn.prepare( + r#" + SELECT payload + FROM account_evidence_checkpoints + WHERE source_id = ?1 + ORDER BY artifact_path_hash, parser_version + "#, + )?; + let rows = statement.query_map([&source_id.0], |row| row.get::<_, String>(0))?; + rows.map(|row| Ok(serde_json::from_str(&row?)?)).collect() + } + + pub fn upsert_account_evidence_checkpoints( + &self, + checkpoints: &[AccountEvidenceCheckpointV1], + ) -> Result { + if checkpoints.is_empty() { + return Ok(0); + } + self.with_immediate_transaction(|| { + let mut written = 0u64; + let updated_at = chrono::Utc::now().to_rfc3339(); + let mut statement = self.conn.prepare( + r#" + INSERT INTO account_evidence_checkpoints ( + source_id, artifact_path_hash, parser_version, maximum_row_id, + checkpoint_row_fingerprint, database_size, database_modified_nanos, + wal_size, wal_modified_nanos, payload, updated_at + ) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) + ON CONFLICT(source_id, artifact_path_hash, parser_version) DO UPDATE SET + maximum_row_id = excluded.maximum_row_id, + checkpoint_row_fingerprint = excluded.checkpoint_row_fingerprint, + database_size = excluded.database_size, + database_modified_nanos = excluded.database_modified_nanos, + wal_size = excluded.wal_size, + wal_modified_nanos = excluded.wal_modified_nanos, + payload = excluded.payload, + updated_at = excluded.updated_at + "#, + )?; + for checkpoint in checkpoints { + written += statement.execute(params![ + &checkpoint.source_id.0, + &checkpoint.artifact_path_hash, + &checkpoint.parser_version, + checkpoint.maximum_row_id, + &checkpoint.checkpoint_row_fingerprint, + checkpoint.database_size, + checkpoint.database_modified_nanos, + checkpoint.wal_size, + checkpoint.wal_modified_nanos, + serde_json::to_string(checkpoint)?, + &updated_at, + ])? as u64; + } + Ok(written) + }) + } + + pub fn account_evidence_reference_counts( + &self, + provider: &str, + provider_account_id: &ProviderAccountId, + ) -> Result { + let count = |table: &str| -> Result { + let value = self.conn.query_row( + &format!( + "SELECT COUNT(*) FROM {table} WHERE provider = ?1 AND provider_account_id = ?2" + ), + params![provider, &provider_account_id.0], + |row| row.get::<_, i64>(0), + )?; + Ok(usize::try_from(value).unwrap_or(usize::MAX)) + }; + Ok(AccountEvidenceReferenceCounts { + identity_observations: count("account_identity_observations")?, + plan_observations: count("account_plan_observations")?, + conversation_bindings: count("conversation_account_bindings")?, + }) + } + + /// Moves every local evidence contribution to a canonical account reference. + /// + /// Plan and conversation identifiers encode the account ID, so they are regenerated while + /// identity observations retain their account-independent deterministic identifiers. + pub fn rekey_account_evidence( + &self, + provider: &str, + from_provider_account_id: &ProviderAccountId, + target_provider_account_id: &ProviderAccountId, + ) -> Result { + self.with_immediate_transaction(|| { + let identities = self + .account_identity_observations(None)? + .into_iter() + .filter(|observation| observation.provider == provider) + .filter(|observation| { + observation.provider_account_id.as_ref() == Some(from_provider_account_id) + }) + .collect::>(); + for mut observation in identities.iter().cloned() { + observation.provider_account_id = Some(target_provider_account_id.clone()); + self.conn.execute( + r#" + UPDATE account_identity_observations + SET provider_account_id = ?1, payload = ?2 + WHERE observation_id = ?3 + "#, + params![ + &target_provider_account_id.0, + serde_json::to_string(&observation)?, + &observation.observation_id, + ], + )?; + } + + let plans = self + .account_plan_observations()? + .into_iter() + .filter(|observation| observation.provider == provider) + .filter(|observation| { + observation.provider_account_id.as_ref() == Some(from_provider_account_id) + }) + .collect::>(); + for mut observation in plans.iter().cloned() { + let previous_id = observation.observation_id.clone(); + observation.provider_account_id = Some(target_provider_account_id.clone()); + observation.observation_id = account_plan_observation_id( + &observation.source_id, + Some(target_provider_account_id), + &observation.raw_plan_name, + observation.observed_at, + observation.evidence_kind, + ); + self.upsert_account_plan_observations(std::slice::from_ref(&observation))?; + if previous_id != observation.observation_id { + self.conn.execute( + "DELETE FROM account_plan_observations WHERE observation_id = ?1", + [&previous_id], + )?; + } + } + + let bindings = self + .conversation_account_bindings(None)? + .into_iter() + .filter(|binding| binding.provider == provider) + .filter(|binding| binding.provider_account_id == *from_provider_account_id) + .collect::>(); + for mut binding in bindings.iter().cloned() { + let previous_id = binding.binding_id.clone(); + binding.provider_account_id = target_provider_account_id.clone(); + binding.binding_id = conversation_account_binding_id( + &binding.source_id, + &binding.conversation_id_hash, + binding.turn_id_hash.as_deref(), + target_provider_account_id, + ); + self.upsert_conversation_account_bindings(std::slice::from_ref(&binding))?; + if previous_id != binding.binding_id { + self.conn.execute( + "DELETE FROM conversation_account_bindings WHERE binding_id = ?1", + [&previous_id], + )?; + } + } + + Ok(AccountEvidenceReferenceCounts { + identity_observations: identities.len(), + plan_observations: plans.len(), + conversation_bindings: bindings.len(), + }) + }) + } + + /// Removes evidence records already present in the append-only ledger. + /// + /// Collectors can safely rediscover deterministic records, but filtering before the scanner + /// decides whether it has work avoids opening a write transaction for an unchanged artifact. + pub fn retain_unseen_account_evidence( + &self, + source_id: &SourceId, + identity_observations: &mut Vec, + plan_observations: &mut Vec, + conversation_bindings: &mut Vec, + ) -> Result<()> { + let load_ids = |table: &str, id_column: &str| -> Result> { + let mut statement = self.conn.prepare(&format!( + "SELECT {id_column} FROM {table} WHERE source_id = ?1" + ))?; + let rows = statement.query_map([&source_id.0], |row| row.get::<_, String>(0))?; + rows.map(|row| Ok(row?)).collect() + }; + let identity_ids = load_ids("account_identity_observations", "observation_id")?; + let plan_ids = load_ids("account_plan_observations", "observation_id")?; + let binding_ids = load_ids("conversation_account_bindings", "binding_id")?; + identity_observations + .retain(|observation| !identity_ids.contains(&observation.observation_id)); + plan_observations.retain(|observation| !plan_ids.contains(&observation.observation_id)); + conversation_bindings.retain(|binding| !binding_ids.contains(&binding.binding_id)); + Ok(()) + } + + /// Deletes every local identity/plan evidence row owned by the supplied scanner sources. + /// + /// This deliberately includes direct conversation bindings: their hashed locators remain + /// source-scoped personal data even when the corresponding usage events have been removed. + pub fn delete_account_evidence_for_sources(&self, source_ids: &[SourceId]) -> Result { + self.with_immediate_transaction(|| { + let mut deleted = 0u64; + for source_id in source_ids { + for table in [ + "account_identity_observations", + "account_plan_observations", + "conversation_account_bindings", + "account_evidence_checkpoints", + ] { + deleted = deleted.saturating_add(self.conn.execute( + &format!("DELETE FROM {table} WHERE source_id = ?1"), + [&source_id.0], + )? as u64); + } + } + Ok(deleted) + }) + } + + /// Converts historical Codex subscriptions and account-level plan fields synthesized from + /// local authentication into provider-plan evidence. User-entered billing records are + /// deliberately left untouched. + /// + /// Conversion and retirement happen in one transaction so a failed evidence write can never + /// discard the legacy record. The deterministic observation ID makes this safe to repeat. + pub fn migrate_legacy_codex_local_auth_subscriptions_to_plan_evidence(&self) -> Result { + self.with_immediate_transaction(|| { + let subscriptions = self.list_subscriptions()?; + let mut converted = 0u64; + for subscription in subscriptions.into_iter().filter(|subscription| { + subscription.provider.eq_ignore_ascii_case("codex") + && subscription.record_source == IdentitySource::LocalAuth + }) { + let source_id = SourceId(format!( + "legacy_local_auth_{}", + &hash_text(&format!( + "legacy_local_auth_source.v1:{}:{}", + subscription.provider, subscription.provider_account_id.0 + ))[..32] + )); + let observed_at = subscription + .verified_at + .or(subscription.paid_at) + .unwrap_or(subscription.started_at); + let record_fingerprint = hash_text(&format!( + "legacy_local_auth_subscription.v1:{}", + subscription.subscription_id.0 + )); + let observation = AccountPlanObservationV1 { + schema_version: ACCOUNT_PLAN_OBSERVATION_SCHEMA_VERSION.to_string(), + observation_id: account_plan_observation_id( + &source_id, + Some(&subscription.provider_account_id), + &subscription.plan_name, + observed_at, + AccountEvidenceKind::LegacyLocalAuth, + ), + provider: subscription.provider.clone(), + source_id, + provider_account_id: Some(subscription.provider_account_id.clone()), + raw_plan_name: subscription.plan_name.clone(), + plan_name: normalize_plan_name(&subscription.plan_name), + observed_at, + active_from: Some(subscription.started_at), + active_until: subscription + .current_period_ends_at + .or(subscription.ended_at), + is_current_snapshot: false, + evidence_kind: AccountEvidenceKind::LegacyLocalAuth, + confidence: Confidence::Medium, + parser_version: "legacy-local-auth-migration.v1".to_string(), + artifact_path_hash: hash_text("legacy_local_auth_subscription"), + record_fingerprint, + }; + self.upsert_account_plan_observations(&[observation])?; + self.delete_subscription(&subscription.subscription_id)?; + converted += 1; + } + + let mut preserved_plans = self + .account_plan_observations()? + .into_iter() + .filter(|observation| { + observation.evidence_kind == AccountEvidenceKind::LegacyLocalAuth + }) + .filter_map(|observation| { + observation.provider_account_id.map(|account_id| { + (account_id, observation.raw_plan_name.trim().to_lowercase()) + }) + }) + .collect::>(); + for mut account in self + .list_accounts()? + .into_iter() + .filter(|account| account.provider.eq_ignore_ascii_case("codex")) + { + let Some(raw_plan_name) = account.plan_name.take() else { + continue; + }; + let raw_plan_name = raw_plan_name.trim().to_string(); + let plan_key = ( + account.provider_account_id.clone(), + raw_plan_name.to_lowercase(), + ); + if !raw_plan_name.is_empty() && !preserved_plans.contains(&plan_key) { + let source_id = SourceId(format!( + "legacy_local_auth_{}", + &hash_text(&format!( + "legacy_local_auth_source.v1:{}:{}", + account.provider, account.provider_account_id.0 + ))[..32] + )); + let observed_at = account.verified_at.unwrap_or(account.updated_at); + let observation = AccountPlanObservationV1 { + schema_version: ACCOUNT_PLAN_OBSERVATION_SCHEMA_VERSION.to_string(), + observation_id: account_plan_observation_id( + &source_id, + Some(&account.provider_account_id), + &raw_plan_name, + observed_at, + AccountEvidenceKind::LegacyLocalAuth, + ), + provider: account.provider.clone(), + source_id, + provider_account_id: Some(account.provider_account_id.clone()), + raw_plan_name: raw_plan_name.clone(), + plan_name: normalize_plan_name(&raw_plan_name), + observed_at, + active_from: None, + active_until: None, + is_current_snapshot: false, + evidence_kind: AccountEvidenceKind::LegacyLocalAuth, + confidence: Confidence::Medium, + parser_version: "legacy-account-plan-migration.v1".to_string(), + artifact_path_hash: hash_text("legacy_provider_account_plan"), + record_fingerprint: hash_text(&format!( + "legacy_provider_account_plan.v1:{}:{}", + account.provider_account_id.0, raw_plan_name + )), + }; + self.upsert_account_plan_observations(&[observation])?; + preserved_plans.insert(plan_key); + } + self.upsert_account(&account)?; + } + Ok(converted) + }) + } + + /// Repairs automatic source-account intervals from explicit auth boundaries while keeping + /// unsupported history unattributed. Manual intervals are immutable here; conflicting direct + /// evidence remains available for conversation- or turn-specific attribution. + pub fn reconcile_source_account_evidence_assignments( + &self, + source_id: &SourceId, + ) -> Result { + self.with_immediate_transaction(|| { + let Some(source) = self.source(source_id)? else { + return Ok(0); + }; + let mut strong = self + .account_identity_observations(Some(source_id))? + .into_iter() + .filter(|observation| { + observation.evidence_kind.is_strong_identity() + && observation.provider_account_id.is_some() + }) + .collect::>(); + strong.sort_by(|left, right| { + left.observed_at + .cmp(&right.observed_at) + .then_with(|| left.observation_id.cmp(&right.observation_id)) + }); + if strong.is_empty() { + return Ok(0); + } + + let mut changed = 0u64; + // A contradictory strong point invalidates generated continuity from that point. The + // previous account is not resumed unless a later explicit boundary supports it. + for observation in &strong { + let account_id = observation + .provider_account_id + .as_ref() + .expect("filtered account identity"); + for mut assignment in self.list_source_account_assignments_for_source(source_id)? { + if assignment.record_source == IdentitySource::UserConfigured + || !is_verified_source_assignment(&assignment) + || assignment.provider_account_id == *account_id + || observation.observed_at < assignment.started_at + || assignment + .ended_at + .is_some_and(|ended_at| observation.observed_at >= ended_at) + { + continue; + } + if observation.observed_at == assignment.started_at { + changed += self + .delete_source_account_assignment(&assignment.assignment_id)? + as u64; + } else { + validate_time_window( + assignment.started_at, + Some(observation.observed_at), + "source connection", + )?; + assignment.ended_at = Some(observation.observed_at); + assignment.updated_at = chrono::Utc::now(); + self.upsert_source_account_assignment(&assignment)?; + changed += 1; + } + } + } + + // A reload is an explicit boundary, but only a later strong point for the same account + // promotes it into a source interval. Current auth snapshots are handled by the + // existing verified-auth reconciliation path. + for (index, boundary) in strong.iter().enumerate().filter(|(_, observation)| { + observation.evidence_kind == AccountEvidenceKind::AuthReload + }) { + let account_id = boundary + .provider_account_id + .as_ref() + .expect("filtered account identity"); + let confirmation = strong[index + 1..].iter().find(|observation| { + observation.provider_account_id.as_ref() == Some(account_id) + }); + let Some(confirmation) = confirmation else { + continue; + }; + let ended_at = strong[index + 1..] + .iter() + .find(|observation| { + observation.provider_account_id.as_ref() != Some(account_id) + }) + .map(|observation| observation.observed_at); + let assignments = self.list_source_account_assignments_for_source(source_id)?; + if assignments.iter().any(|assignment| { + assignment.record_source == IdentitySource::UserConfigured + && periods_overlap( + boundary.observed_at, + ended_at, + assignment.started_at, + assignment.ended_at, + ) + }) { + continue; + } + let overlapping_same = assignments.iter().find(|assignment| { + assignment.provider_account_id == *account_id + && is_verified_source_assignment(assignment) + && periods_overlap( + boundary.observed_at, + ended_at, + assignment.started_at, + assignment.ended_at, + ) + }); + if overlapping_same.is_some_and(|existing| { + existing.started_at <= boundary.observed_at + && existing.ended_at == ended_at + && existing + .verified_at + .is_some_and(|verified_at| verified_at >= confirmation.observed_at) + }) { + continue; + } + let now = chrono::Utc::now(); + let (started_at, merged_end, created_at, previous_id) = overlapping_same.map_or( + (boundary.observed_at, ended_at, now, None), + |existing| { + let merged_end = match (existing.ended_at, ended_at) { + (None, _) | (_, None) => None, + (Some(left), Some(right)) => Some(left.max(right)), + }; + ( + existing.started_at.min(boundary.observed_at), + merged_end, + existing.created_at, + Some(existing.assignment_id.clone()), + ) + }, + ); + let assignment = SourceAccountAssignment { + schema_version: SOURCE_ACCOUNT_ASSIGNMENT_SCHEMA_VERSION.to_string(), + assignment_id: source_account_assignment_id(source_id, account_id, started_at), + source_id: source_id.clone(), + provider: source.provider.clone(), + provider_account_id: account_id.clone(), + started_at, + ended_at: merged_end, + record_source: IdentitySource::LocalAuth, + verified_at: Some(confirmation.observed_at), + created_at, + updated_at: now, + }; + if let Some(previous_id) = + previous_id.filter(|previous_id| *previous_id != assignment.assignment_id) + { + self.delete_source_account_assignment(&previous_id)?; + } + self.upsert_source_account_assignment(&assignment)?; + changed += 1; + } + if changed > 0 { + reattribute_source_records(self, source_id)?; + } + Ok(changed) + }) + } + + /// Convert an attributed quota status into plan evidence. A plan label by itself never + /// identifies an account, so records without an unambiguous source assignment are skipped. + pub fn upsert_quota_plan_observations( + &self, + records: &[QuotaObservationRecordV1], + ) -> Result { + let mut assignments_by_source = HashMap::new(); + let mut observations = Vec::new(); + let mut seen = HashSet::new(); + for record in records { + let quota = &record.observation; + let Some(raw_plan_name) = quota + .status + .plan_type + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + else { + continue; + }; + if !assignments_by_source.contains_key("a.source_id) { + assignments_by_source.insert( + quota.source_id.clone(), + self.list_source_account_assignments_for_source("a.source_id)?, + ); + } + let provider_account_id = quota.provider_account_id.clone().or_else(|| { + assignments_by_source + .get("a.source_id) + .and_then(|assignments| { + assignment_for_timestamp(assignments, quota.observed_at) + }) + .map(|assignment| assignment.provider_account_id.clone()) + }); + let Some(provider_account_id) = provider_account_id else { + continue; + }; + let observation_id = account_plan_observation_id( + "a.source_id, + Some(&provider_account_id), + raw_plan_name, + quota.observed_at, + AccountEvidenceKind::QuotaStatus, + ); + if !seen.insert(observation_id.clone()) { + continue; + } + observations.push(AccountPlanObservationV1 { + schema_version: ACCOUNT_PLAN_OBSERVATION_SCHEMA_VERSION.to_string(), + observation_id, + provider: quota.provider.clone(), + source_id: quota.source_id.clone(), + provider_account_id: Some(provider_account_id), + raw_plan_name: raw_plan_name.to_string(), + plan_name: normalize_plan_name(raw_plan_name), + observed_at: quota.observed_at, + active_from: None, + active_until: None, + is_current_snapshot: false, + evidence_kind: AccountEvidenceKind::QuotaStatus, + confidence: Confidence::High, + parser_version: "quota-plan-evidence.v1".to_string(), + artifact_path_hash: quota.source_file_path_hash.clone(), + record_fingerprint: quota.semantic_fingerprint.clone(), + }); + } + self.upsert_account_plan_observations(&observations) + } + + /// Rebuilds every plan observation derived from quota status for one source. + /// + /// Quota rows are mutable scan projections, while the general plan ledger is append-only. + /// Removing the old derived subset before recreating it prevents corrected files or changed + /// account attribution from leaving stale plan/account claims behind. + pub fn rebuild_quota_plan_observations_for_source(&self, source_id: &SourceId) -> Result { + self.with_immediate_transaction(|| { + self.conn.execute( + "DELETE FROM account_plan_observations WHERE source_id = ?1 AND evidence_kind = ?2", + params![ + &source_id.0, + serde_json::to_string(&AccountEvidenceKind::QuotaStatus)? + ], + )?; + let records = self.quota_observations_for_source(source_id)?; + self.upsert_quota_plan_observations(&records) + }) + } + + pub fn upsert_account_identity_observations( + &self, + observations: &[AccountIdentityObservationV1], + ) -> Result { + if observations.is_empty() { + return Ok(0); + } + self.with_immediate_transaction(|| { + let mut written = 0u64; + let mut statement = self.conn.prepare( + r#" + INSERT INTO account_identity_observations ( + observation_id, provider, source_id, provider_account_id, observed_at, + evidence_kind, conversation_id_hash, payload + ) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) + ON CONFLICT(observation_id) DO NOTHING + "#, + )?; + for observation in observations { + written += statement.execute(params![ + &observation.observation_id, + &observation.provider, + &observation.source_id.0, + observation + .provider_account_id + .as_ref() + .map(|value| value.0.as_str()), + observation.observed_at.to_rfc3339(), + serde_json::to_string(&observation.evidence_kind)?, + observation.conversation_id_hash.as_deref(), + serde_json::to_string(observation)?, + ])? as u64; + } + Ok(written) + }) + } + + pub fn account_identity_observations( + &self, + source_id: Option<&SourceId>, + ) -> Result> { + let mut statement = self.conn.prepare( + r#" + SELECT payload + FROM account_identity_observations + WHERE (?1 IS NULL OR source_id = ?1) + ORDER BY observed_at, observation_id + "#, + )?; + let rows = statement + .query_map(params![source_id.map(|value| value.0.as_str())], |row| { + row.get::<_, String>(0) + })?; + rows.map(|row| Ok(serde_json::from_str(&row?)?)).collect() + } + + pub fn upsert_account_plan_observations( + &self, + observations: &[AccountPlanObservationV1], + ) -> Result { + if observations.is_empty() { + return Ok(0); + } + self.with_immediate_transaction(|| { + let mut written = 0u64; + let mut statement = self.conn.prepare( + r#" + INSERT INTO account_plan_observations ( + observation_id, provider, source_id, provider_account_id, observed_at, + active_from, active_until, plan_name, evidence_kind, payload + ) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10) + ON CONFLICT(observation_id) DO NOTHING + "#, + )?; + for observation in observations { + written += statement.execute(params![ + &observation.observation_id, + &observation.provider, + &observation.source_id.0, + observation + .provider_account_id + .as_ref() + .map(|value| value.0.as_str()), + observation.observed_at.to_rfc3339(), + observation.active_from.map(|value| value.to_rfc3339()), + observation.active_until.map(|value| value.to_rfc3339()), + &observation.plan_name, + serde_json::to_string(&observation.evidence_kind)?, + serde_json::to_string(observation)?, + ])? as u64; + } + Ok(written) + }) + } + + pub fn account_plan_observations(&self) -> Result> { + let mut statement = self.conn.prepare( + "SELECT payload FROM account_plan_observations ORDER BY observed_at, observation_id", + )?; + let rows = statement.query_map([], |row| row.get::<_, String>(0))?; + rows.map(|row| Ok(serde_json::from_str(&row?)?)).collect() + } + + pub fn upsert_conversation_account_bindings( + &self, + bindings: &[ConversationAccountBindingV1], + ) -> Result { + if bindings.is_empty() { + return Ok(0); + } + self.with_immediate_transaction(|| { + let mut written = 0u64; + let mut statement = self.conn.prepare( + r#" + INSERT INTO conversation_account_bindings ( + binding_id, provider, source_id, provider_account_id, + conversation_id_hash, turn_id_hash, observed_at, payload + ) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) + ON CONFLICT(binding_id) DO NOTHING + "#, + )?; + for binding in bindings { + written += statement.execute(params![ + &binding.binding_id, + &binding.provider, + &binding.source_id.0, + &binding.provider_account_id.0, + &binding.conversation_id_hash, + binding.turn_id_hash.as_deref(), + binding.observed_at.to_rfc3339(), + serde_json::to_string(binding)?, + ])? as u64; + } + Ok(written) + }) + } + + pub fn conversation_account_bindings( + &self, + source_id: Option<&SourceId>, + ) -> Result> { + let mut statement = self.conn.prepare( + r#" + SELECT payload + FROM conversation_account_bindings + WHERE (?1 IS NULL OR source_id = ?1) + ORDER BY observed_at, binding_id + "#, + )?; + let rows = statement + .query_map(params![source_id.map(|value| value.0.as_str())], |row| { + row.get::<_, String>(0) + })?; + rows.map(|row| Ok(serde_json::from_str(&row?)?)).collect() + } + + pub fn apply_conversation_account_bindings( + &self, + source_id: &SourceId, + events: &mut [UsageEvent], + ) -> Result<()> { + let bindings = self.conversation_account_bindings(Some(source_id))?; + let mut accounts_by_conversation: HashMap<&str, HashSet<&ProviderAccountId>> = + HashMap::new(); + for binding in bindings + .iter() + // A turn-scoped reset-history record cannot safely relabel every event in its + // conversation. Usage events currently expose only conversation identity, so retain + // that binding as evidence until an exact turn locator is available. + .filter(|binding| binding.turn_id_hash.is_none()) + { + accounts_by_conversation + .entry(binding.conversation_id_hash.as_str()) + .or_default() + .insert(&binding.provider_account_id); + } + for event in events { + let Some(conversation_id_hash) = event.session.local_session_id_hash.as_deref() else { + continue; + }; + let Some(accounts) = accounts_by_conversation.get(conversation_id_hash) else { + continue; + }; + if accounts.len() != 1 { + event.provider_account_id = None; + continue; + } + event.provider_account_id = accounts.iter().next().map(|value| (*value).clone()); + if let Some(parse_evidence) = event.parse_evidence.as_mut() { + parse_evidence.account_identity_source = IdentitySource::LocalAuth; + } + } + Ok(()) + } + + pub fn reattribute_conversation_bound_events(&self, source_id: &SourceId) -> Result { + let mut events = self + .events()? + .into_iter() + .filter(|event| event.source_id == *source_id) + .collect::>(); + let previous_accounts = events + .iter() + .map(|event| event.provider_account_id.clone()) + .collect::>(); + self.apply_conversation_account_bindings(source_id, &mut events)?; + let mut changed = 0u64; + let mut dirty_keys = BTreeSet::new(); + for (event, previous_account) in events.iter().zip(previous_accounts) { + if event.provider_account_id == previous_account { + continue; + } + dirty_keys.extend(self.update_event_payload(event)?); + changed += 1; + } + self.refresh_sync_rollups_for_keys(&dirty_keys)?; + Ok(changed) + } + + pub fn account_plan_projections( + &self, + device_id: &str, + ) -> Result> { + Ok(self + .account_plan_observations()? + .iter() + .filter_map(|observation| plan_projection_from_observation(observation, device_id)) + .collect()) + } + + pub fn account_evidence_summaries( + &self, + device_id: &str, + ) -> Result> { + let observations = self.account_identity_observations(None)?; + let bindings = self.conversation_account_bindings(None)?; + let mut conversations_by_account = HashMap::>::new(); + let mut accounts_by_conversation = HashMap::>::new(); + for binding in &bindings { + conversations_by_account + .entry(binding.provider_account_id.clone()) + .or_default() + .insert(binding.conversation_id_hash.clone()); + accounts_by_conversation + .entry(binding.conversation_id_hash.clone()) + .or_default() + .insert(binding.provider_account_id.clone()); + } + let mut conflicting_conversations_by_account = HashMap::::new(); + for accounts in accounts_by_conversation + .values() + .filter(|accounts| accounts.len() > 1) + { + for account in accounts { + *conflicting_conversations_by_account + .entry(account.clone()) + .or_default() += 1; + } + } + let mut assignments_by_source = HashMap::new(); + for observation in &observations { + if !assignments_by_source.contains_key(&observation.source_id) { + assignments_by_source.insert( + observation.source_id.clone(), + self.list_source_account_assignments_for_source(&observation.source_id)?, + ); + } + } + let mut grouped: BTreeMap<(String, ProviderAccountId), Vec<&AccountIdentityObservationV1>> = + BTreeMap::new(); + for observation in &observations { + let Some(provider_account_id) = observation.provider_account_id.clone() else { + continue; + }; + grouped + .entry((observation.provider.clone(), provider_account_id)) + .or_default() + .push(observation); + } + let mut summaries = Vec::with_capacity(grouped.len()); + for ((provider, provider_account_id), observations) in grouped { + let strong = observations + .iter() + .filter(|observation| observation.evidence_kind.is_strong_identity()) + .copied() + .collect::>(); + let evidence_kinds = observations + .iter() + .map(|observation| observation.evidence_kind) + .collect::>() + .into_iter() + .collect(); + let directly_bound_conversations = conversations_by_account + .get(&provider_account_id) + .map_or(0, |conversations| conversations.len() as u64); + let conflicting_conversations = conflicting_conversations_by_account + .get(&provider_account_id) + .copied() + .unwrap_or_default(); + let mut uncovered_gap_count = 0u64; + let mut assignment_conflict_count = 0u64; + let mut previous_uncovered_by_source = HashMap::<&SourceId, bool>::new(); + for observation in &strong { + let directly_bound = observation.conversation_id_hash.as_deref().is_some_and( + |conversation_id_hash| { + conversations_by_account + .get(&provider_account_id) + .is_some_and(|conversations| { + conversations.contains(conversation_id_hash) + }) + }, + ); + let assignments = assignments_by_source + .get(&observation.source_id) + .map(Vec::as_slice) + .unwrap_or_default(); + let active = assignments + .iter() + .filter(|assignment| { + observation.observed_at >= assignment.started_at + && assignment + .ended_at + .is_none_or(|ended_at| observation.observed_at < ended_at) + }) + .collect::>(); + let has_matching_assignment = active + .iter() + .any(|assignment| assignment.provider_account_id == provider_account_id); + let has_conflicting_assignment = active + .iter() + .any(|assignment| assignment.provider_account_id != provider_account_id); + if has_conflicting_assignment && !has_matching_assignment { + assignment_conflict_count += 1; + } + let uncovered = !directly_bound && !has_matching_assignment; + let previous_uncovered = previous_uncovered_by_source + .insert(&observation.source_id, uncovered) + .unwrap_or(false); + if uncovered && !previous_uncovered { + uncovered_gap_count += 1; + } + } + let summary_id = format!( + "account_evidence_summary_{}", + &statsai_core::hash_text(&format!( + "account_evidence_summary.v1:{device_id}:{provider}:{}", + provider_account_id.0 + ))[..32] + ); + summaries.push(AccountEvidenceSummaryV1 { + schema_version: ACCOUNT_EVIDENCE_SUMMARY_SCHEMA_VERSION.to_string(), + summary_id, + device_id: device_id.to_string(), + provider, + provider_account_id, + first_strong_observed_at: strong.iter().map(|value| value.observed_at).min(), + last_strong_observed_at: strong.iter().map(|value| value.observed_at).max(), + strong_observation_count: strong.len() as u64, + directly_bound_conversations, + uncovered_gap_count, + conflict_count: conflicting_conversations + assignment_conflict_count, + evidence_kinds, + }); + } + Ok(summaries) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use statsai_core::{ + LocationOrigin, SourceLocation, ACCOUNT_EVIDENCE_CHECKPOINT_SCHEMA_VERSION, + ACCOUNT_IDENTITY_OBSERVATION_SCHEMA_VERSION, CONVERSATION_ACCOUNT_BINDING_SCHEMA_VERSION, + }; + use std::path::Path; + + #[test] + fn source_evidence_cleanup_removes_identity_plan_and_conversation_rows() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/account-evidence-cleanup"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let observed_at = Utc::now(); + let account_id = ProviderAccountId("account-cleanup".to_string()); + let identity = AccountIdentityObservationV1 { + schema_version: ACCOUNT_IDENTITY_OBSERVATION_SCHEMA_VERSION.to_string(), + observation_id: "identity-cleanup".to_string(), + provider: "codex".to_string(), + source_id: source.source_id.clone(), + provider_account_id: Some(account_id.clone()), + provider_user_id_hash: Some("a".repeat(64)), + email_hash: None, + conversation_id_hash: Some("b".repeat(64)), + turn_id_hash: None, + observed_at, + evidence_kind: AccountEvidenceKind::TelemetryIdentity, + confidence: Confidence::High, + auth_mode: Some("chatgpt".to_string()), + application_version: None, + parser_version: "test.v1".to_string(), + artifact_kind: "test".to_string(), + artifact_path_hash: "c".repeat(64), + record_fingerprint: "d".repeat(64), + }; + let plan = AccountPlanObservationV1 { + schema_version: ACCOUNT_PLAN_OBSERVATION_SCHEMA_VERSION.to_string(), + observation_id: "plan-cleanup".to_string(), + provider: "codex".to_string(), + source_id: source.source_id.clone(), + provider_account_id: Some(account_id.clone()), + raw_plan_name: "pro".to_string(), + plan_name: "Pro".to_string(), + observed_at, + active_from: None, + active_until: None, + is_current_snapshot: false, + evidence_kind: AccountEvidenceKind::QuotaStatus, + confidence: Confidence::High, + parser_version: "test.v1".to_string(), + artifact_path_hash: "c".repeat(64), + record_fingerprint: "e".repeat(64), + }; + let binding = ConversationAccountBindingV1 { + schema_version: CONVERSATION_ACCOUNT_BINDING_SCHEMA_VERSION.to_string(), + binding_id: "binding-cleanup".to_string(), + provider: "codex".to_string(), + source_id: source.source_id.clone(), + provider_account_id: account_id, + conversation_id_hash: "b".repeat(64), + turn_id_hash: None, + observed_at, + evidence_kind: AccountEvidenceKind::ResetHistory, + confidence: Confidence::High, + }; + store + .upsert_account_identity_observations(std::slice::from_ref(&identity)) + .expect("identity evidence"); + store + .upsert_account_plan_observations(std::slice::from_ref(&plan)) + .expect("plan evidence"); + store + .upsert_conversation_account_bindings(std::slice::from_ref(&binding)) + .expect("conversation evidence"); + let checkpoint = AccountEvidenceCheckpointV1 { + schema_version: ACCOUNT_EVIDENCE_CHECKPOINT_SCHEMA_VERSION.to_string(), + source_id: source.source_id.clone(), + artifact_path_hash: "f".repeat(64), + parser_version: "test.v1".to_string(), + maximum_row_id: 42, + checkpoint_row_fingerprint: Some("1".repeat(64)), + database_size: 100, + database_modified_nanos: 200, + wal_size: 10, + wal_modified_nanos: 300, + }; + store + .upsert_account_evidence_checkpoints(std::slice::from_ref(&checkpoint)) + .expect("checkpoint"); + + let mut identities = vec![identity]; + let mut plans = vec![plan]; + let mut bindings = vec![binding]; + store + .retain_unseen_account_evidence( + &source.source_id, + &mut identities, + &mut plans, + &mut bindings, + ) + .expect("filter known evidence"); + assert!(identities.is_empty() && plans.is_empty() && bindings.is_empty()); + + assert_eq!( + store + .delete_account_evidence_for_sources(std::slice::from_ref(&source.source_id)) + .expect("delete source evidence"), + 4 + ); + assert!(store + .account_identity_observations(Some(&source.source_id)) + .expect("identities") + .is_empty()); + assert!(store.account_plan_observations().expect("plans").is_empty()); + assert!(store + .conversation_account_bindings(Some(&source.source_id)) + .expect("bindings") + .is_empty()); + assert!(store + .account_evidence_checkpoints(&source.source_id) + .expect("checkpoints") + .is_empty()); + } + + #[test] + fn account_evidence_checkpoint_persists_and_rolls_back_transactionally() { + let directory = tempfile::tempdir().expect("temporary directory"); + let path = directory.path().join("statsai.sqlite"); + let source_id = SourceId("checkpoint-source".to_string()); + let checkpoint = AccountEvidenceCheckpointV1 { + schema_version: ACCOUNT_EVIDENCE_CHECKPOINT_SCHEMA_VERSION.to_string(), + source_id: source_id.clone(), + artifact_path_hash: "a".repeat(64), + parser_version: "test.v1".to_string(), + maximum_row_id: 84, + checkpoint_row_fingerprint: Some("2".repeat(64)), + database_size: 200, + database_modified_nanos: 300, + wal_size: 20, + wal_modified_nanos: 400, + }; + { + let store = Store::open(&path).expect("open store"); + store + .upsert_account_evidence_checkpoints(std::slice::from_ref(&checkpoint)) + .expect("persist checkpoint"); + } + let store = Store::open(&path).expect("reopen store"); + assert_eq!( + store + .account_evidence_checkpoints(&source_id) + .expect("load checkpoint"), + vec![checkpoint.clone()] + ); + + let replacement = AccountEvidenceCheckpointV1 { + maximum_row_id: 100, + ..checkpoint.clone() + }; + let error = store + .apply_scan_update(|store| -> Result<()> { + store.upsert_account_evidence_checkpoints(std::slice::from_ref(&replacement))?; + anyhow::bail!("force rollback") + }) + .expect_err("transaction must fail"); + assert_eq!(error.to_string(), "force rollback"); + assert_eq!( + store + .account_evidence_checkpoints(&source_id) + .expect("load checkpoint after rollback"), + vec![checkpoint] + ); + } +} diff --git a/crates/statsai-store/src/code_changes.rs b/crates/statsai-store/src/code_changes.rs index 5c70b4e..08a76ce 100644 --- a/crates/statsai-store/src/code_changes.rs +++ b/crates/statsai-store/src/code_changes.rs @@ -2161,6 +2161,8 @@ mod tests { accounts: Vec::new(), source_account_assignments: Vec::new(), subscriptions: Vec::new(), + account_plan_observations: Vec::new(), + account_evidence_summaries: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: Vec::new(), diff --git a/crates/statsai-store/src/lib.rs b/crates/statsai-store/src/lib.rs index 2bb135d..7f2b324 100644 --- a/crates/statsai-store/src/lib.rs +++ b/crates/statsai-store/src/lib.rs @@ -1,5 +1,6 @@ //! Local SQLite storage for `statsai`. +mod account_plan; mod archive; mod code_changes; mod migrations; @@ -11,15 +12,16 @@ mod tasks; use anyhow::{bail, Context, Result}; use chrono::{DateTime, Datelike, NaiveDate, Utc}; use rusqlite::{params, Connection, OptionalExtension}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use statsai_core::{ hash_text, micro_usd_to_cents_rounded, normalize_email, normalize_provider_user_id, periods_overlap, project_bucket_key, project_contains_file_paths, project_has_stable_identity, provider_account_id, provider_account_id_from_identity, sanitize_code_change_metric_for_sync, sanitize_summary_for_sync, semantic_event_fingerprint, source_account_assignment_id, - subscription_id, summary_id, timestamp_in_period, BillingPeriod, CodeChangeMetric, Confidence, - CostAccumulator, CostInfo, DailyRollup, EventId, EventSource, IdentitySource, LatencySource, - MetricStats, ModelInfo, PrivacyInfo, PrivacyMode, ProviderAccount, ProviderAccountId, + subscription_id, summary_id, timestamp_in_period, AccountEvidenceSummaryV1, + AccountPlanProjectionV1, BillingPeriod, CodeChangeMetric, Confidence, CostAccumulator, + CostInfo, DailyRollup, EventId, EventSource, IdentitySource, LatencySource, MetricStats, + ModelInfo, PrivacyInfo, PrivacyMode, ProviderAccount, ProviderAccountId, SemanticFingerprintInput, SourceAccountAssignment, SourceAccountAssignmentId, SourceId, SourceKind, SourceLocation, SourceVerificationMode, Subscription, SubscriptionId, SubscriptionStatus, SummaryId, SummaryMetadata, SummaryMetricTotals, SummaryMetrics, @@ -32,6 +34,7 @@ use statsai_core::{ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::Path; +pub use account_plan::AccountEvidenceReferenceCounts; pub use migrations::CURRENT_SCHEMA_VERSION; pub use snapshot::{clone_database_to, database_schema_version, DatabaseClone}; use std::time::Duration; @@ -58,6 +61,7 @@ pub use tasks::{ const SYNC_ROLLUP_SUMMARY_VERSION: &str = "12"; const SYNC_INCLUDE_PROJECTS_METADATA_KEY: &str = "sync.include_projects"; const SYNC_INCLUDE_TASKS_METADATA_KEY: &str = "sync.include_tasks"; +const LEGACY_CODEX_PLAN_CONVERSION_METADATA_KEY: &str = "migration.legacy_codex_plan_evidence.v1"; const SQLITE_BUSY_TIMEOUT: Duration = if cfg!(test) { Duration::from_millis(50) } else { @@ -720,7 +724,26 @@ impl Store { } pub fn migrate(&self) -> Result<()> { - migrations::migrate(&self.conn) + migrations::migrate(&self.conn)?; + if self + .metadata_value(LEGACY_CODEX_PLAN_CONVERSION_METADATA_KEY)? + .as_deref() + != Some("1") + { + self.with_immediate_transaction(|| { + if self + .metadata_value(LEGACY_CODEX_PLAN_CONVERSION_METADATA_KEY)? + .as_deref() + == Some("1") + { + return Ok(()); + } + self.migrate_legacy_codex_local_auth_subscriptions_to_plan_evidence()?; + self.set_metadata_value(LEGACY_CODEX_PLAN_CONVERSION_METADATA_KEY, "1")?; + Ok(()) + })?; + } + Ok(()) } pub fn schema_version(&self) -> Result { @@ -2516,6 +2539,20 @@ impl Store { target, &batch.quota_cycle_contributions, )?; + self.record_serialized_entities_synced_in_transaction( + sink, + target, + "account_plan_observation", + &batch.account_plan_observations, + |projection| projection.projection_id.as_str(), + )?; + self.record_serialized_entities_synced_in_transaction( + sink, + target, + "account_evidence_summary", + &batch.account_evidence_summaries, + |summary| summary.summary_id.as_str(), + )?; self.record_task_bucket_snapshots_synced_in_transaction( sink, target, @@ -2829,6 +2866,60 @@ impl Store { Ok(changed) } + fn pending_serialized_entities_for_sync( + &self, + sink: &str, + target: &str, + entity_kind: &str, + entities: &[T], + entity_id: impl Fn(&T) -> &str, + ) -> Result> { + let mut changed = Vec::new(); + for entity in entities { + let payload = serde_json::to_string(entity)?; + if self.entity_requires_sync( + sink, + target, + entity_kind, + entity_id(entity), + &hash_text(&payload), + )? { + changed.push(entity.clone()); + } + } + Ok(changed) + } + + pub fn pending_account_plan_projections_for_sync( + &self, + sink: &str, + target: &str, + projections: &[AccountPlanProjectionV1], + ) -> Result> { + self.pending_serialized_entities_for_sync( + sink, + target, + "account_plan_observation", + projections, + |projection| projection.projection_id.as_str(), + ) + } + + pub fn pending_account_evidence_summaries_for_sync( + &self, + sink: &str, + target: &str, + summaries: &[AccountEvidenceSummaryV1], + ) -> Result> { + self.pending_serialized_entities_for_sync( + sink, + target, + "account_evidence_summary", + summaries, + |summary| summary.summary_id.as_str(), + ) + } + pub fn pending_summaries_for_sync( &self, sink: &str, @@ -2950,6 +3041,22 @@ impl Store { .map(String::as_str) .collect::>(), ), + ( + "account_plan_observation", + snapshot + .account_plan_observation_ids + .iter() + .map(String::as_str) + .collect::>(), + ), + ( + "account_evidence_summary", + snapshot + .account_evidence_summary_ids + .iter() + .map(String::as_str) + .collect::>(), + ), ]); let mut statement = self.conn.prepare( r#" @@ -2958,7 +3065,8 @@ impl Store { WHERE sink = ?1 AND target = ?2 AND entity_kind IN ( 'source', 'account', 'source_account_assignment', 'subscription', 'summary', - 'code_change_metric', 'quota_cycle_contribution' + 'code_change_metric', 'quota_cycle_contribution', + 'account_plan_observation', 'account_evidence_summary' ) "#, )?; @@ -3035,6 +3143,18 @@ impl Store { )?; let current_quota_cycle_contributions = self.quota_cycle_contributions(&QuotaQuery::default(), device_id)?; + let current_account_plan_observations = self.account_plan_projections(device_id)?; + let account_plan_observations = self.pending_account_plan_projections_for_sync( + "http", + target, + ¤t_account_plan_observations, + )?; + let current_account_evidence_summaries = self.account_evidence_summaries(device_id)?; + let account_evidence_summaries = self.pending_account_evidence_summaries_for_sync( + "http", + target, + ¤t_account_evidence_summaries, + )?; let current_snapshot = self.current_http_sync_authoritative_snapshot( ¤t_rollups, ¤t_passthrough_summaries, @@ -3043,6 +3163,8 @@ impl Store { .iter() .map(|contribution| contribution.contribution_id.clone()) .collect::>(), + ¤t_account_plan_observations, + ¤t_account_evidence_summaries, )?; // A quota cycle can change without any summary changing: a reset moves, // or an observation carries no tokens. Counting only the summary-shaped @@ -3069,6 +3191,8 @@ impl Store { .saturating_add(passthrough_summaries.len()) .saturating_add(code_change_metrics.len()) .saturating_add(quota_cycle_contributions.len()) + .saturating_add(account_plan_observations.len()) + .saturating_add(account_evidence_summaries.len()) .saturating_add(retired_entities) as u64, days: days.len() as u64, }) @@ -3080,6 +3204,8 @@ impl Store { passthrough_summaries: &[UsageSummary], code_change_metrics: &[CodeChangeMetric], quota_cycle_contribution_ids: &[String], + account_plan_observations: &[AccountPlanProjectionV1], + account_evidence_summaries: &[AccountEvidenceSummaryV1], ) -> Result { Ok(SyncAuthoritativeSnapshot { snapshot_id: String::new(), @@ -3115,6 +3241,14 @@ impl Store { .map(|metric| metric.metric_id.clone()) .collect(), quota_cycle_contribution_ids: quota_cycle_contribution_ids.to_vec(), + account_plan_observation_ids: account_plan_observations + .iter() + .map(|observation| observation.projection_id.clone()) + .collect(), + account_evidence_summary_ids: account_evidence_summaries + .iter() + .map(|summary| summary.summary_id.clone()) + .collect(), }) } @@ -3563,6 +3697,67 @@ impl Store { Ok(()) } + fn record_serialized_entities_synced_in_transaction( + &self, + sink: &str, + target: &str, + entity_kind: &str, + entities: &[T], + entity_id: impl Fn(&T) -> &str, + ) -> Result<()> { + for entity in entities { + let payload = serde_json::to_string(entity)?; + self.record_entity_synced( + sink, + target, + entity_kind, + entity_id(entity), + &hash_text(&payload), + )?; + } + Ok(()) + } + + pub fn record_account_plan_projections_synced( + &self, + sink: &str, + target: &str, + projections: &[AccountPlanProjectionV1], + ) -> Result<()> { + if projections.is_empty() { + return Ok(()); + } + self.with_immediate_transaction(|| { + self.record_serialized_entities_synced_in_transaction( + sink, + target, + "account_plan_observation", + projections, + |projection| projection.projection_id.as_str(), + ) + }) + } + + pub fn record_account_evidence_summaries_synced( + &self, + sink: &str, + target: &str, + summaries: &[AccountEvidenceSummaryV1], + ) -> Result<()> { + if summaries.is_empty() { + return Ok(()); + } + self.with_immediate_transaction(|| { + self.record_serialized_entities_synced_in_transaction( + sink, + target, + "account_evidence_summary", + summaries, + |summary| summary.summary_id.as_str(), + ) + }) + } + pub fn compute_daily_rollup(&self, date: &str, device_id: &str) -> Result { let mut stmt = self.conn.prepare( r#" @@ -4184,7 +4379,9 @@ fn apply_verified_source_state_with_recovery_boundary( provider_user_id: verified_state.provider_user_id.as_deref(), email: verified_state.email.as_deref(), label: verified_state.account_label.clone(), - plan_name: verified_state.plan_name.clone(), + plan_name: (!source.provider.eq_ignore_ascii_case("codex")) + .then(|| verified_state.plan_name.clone()) + .flatten(), identity_source: Some(record_source.clone()), verified_at: verified_state.verified_at, }, @@ -4591,6 +4788,11 @@ fn upsert_verified_subscription( provider_account_id: &ProviderAccountId, verified: &VerifiedSubscriptionState, ) -> Result<()> { + // Codex authentication exposes provider plan state, not user billing facts. Its adapter + // persists that information through the plan-evidence ledger instead of this billing table. + if provider.eq_ignore_ascii_case("codex") { + return Ok(()); + } validate_time_window(verified.started_at, None, "subscription")?; let subscriptions: Vec<_> = store .list_subscriptions()? @@ -4882,6 +5084,7 @@ fn reattribute_source_records(store: &Store, source_id: &SourceId) -> Result<()> store.rewrite_events(&events)?; store.rewrite_summaries(&summaries)?; store.reattribute_quota_observations(source_id)?; + store.rebuild_quota_plan_observations_for_source(source_id)?; Ok(()) } @@ -6737,6 +6940,396 @@ mod tests { ); } + #[test] + fn migrates_only_codex_local_auth_subscriptions_into_plan_evidence() { + let store = Store::in_memory().expect("store"); + let started_at = Utc + .with_ymd_and_hms(2026, 6, 1, 0, 0, 0) + .single() + .expect("started_at"); + let active_until = Utc + .with_ymd_and_hms(2026, 7, 1, 0, 0, 0) + .single() + .expect("active_until"); + let account_id = provider_account_id("codex", "provider-user"); + let synthetic = Subscription { + schema_version: SUBSCRIPTION_SCHEMA_VERSION.to_string(), + subscription_id: SubscriptionId("synthetic-codex-plan".to_string()), + provider: "codex".to_string(), + provider_account_id: account_id.clone(), + plan_name: "future_ultra".to_string(), + price: 20_00, + currency: "USD".to_string(), + billing_period: BillingPeriod::Monthly, + paid_at: Some(started_at), + renewal_day: Some(1), + started_at, + ended_at: None, + current_period_ends_at: Some(active_until), + status: SubscriptionStatus::Active, + record_source: IdentitySource::LocalAuth, + verified_at: Some(started_at), + notes: None, + }; + let manual = Subscription { + subscription_id: SubscriptionId("manual-codex-billing".to_string()), + record_source: IdentitySource::UserConfigured, + price: 12_34, + ..synthetic.clone() + }; + store.upsert_subscription(&synthetic).expect("synthetic"); + store.upsert_subscription(&manual).expect("manual"); + + assert_eq!( + store + .migrate_legacy_codex_local_auth_subscriptions_to_plan_evidence() + .expect("migration"), + 1 + ); + assert_eq!( + store + .migrate_legacy_codex_local_auth_subscriptions_to_plan_evidence() + .expect("repeat migration"), + 0 + ); + + assert_eq!(store.list_subscriptions().expect("billing"), vec![manual]); + let observations = store.account_plan_observations().expect("plan evidence"); + assert_eq!(observations.len(), 1); + assert_eq!(observations[0].provider_account_id, Some(account_id)); + assert_eq!(observations[0].raw_plan_name, "future_ultra"); + assert_eq!(observations[0].plan_name, "Future Ultra"); + assert_eq!(observations[0].active_from, Some(started_at)); + assert_eq!(observations[0].active_until, Some(active_until)); + assert_eq!( + observations[0].evidence_kind, + statsai_core::AccountEvidenceKind::LegacyLocalAuth + ); + } + + #[test] + fn migrates_legacy_codex_account_plan_without_subscription() { + let store = Store::in_memory().expect("store"); + let observed_at = Utc + .with_ymd_and_hms(2026, 6, 1, 0, 0, 0) + .single() + .expect("observed at"); + let account_id = provider_account_id("codex", "legacy-plan-account"); + store + .upsert_account(&ProviderAccount { + schema_version: PROVIDER_ACCOUNT_SCHEMA_VERSION.to_string(), + provider_account_id: account_id.clone(), + provider: "codex".to_string(), + identity_source: IdentitySource::LocalAuth, + provider_user_id: None, + provider_user_id_hash: Some("a".repeat(64)), + email: None, + email_hash: None, + org_id_hash: None, + account_label: Some("Codex account".to_string()), + plan_name: Some("future_ultra".to_string()), + confidence: Confidence::High, + verified_at: Some(observed_at), + created_at: observed_at, + updated_at: observed_at, + }) + .expect("legacy account"); + + assert_eq!( + store + .migrate_legacy_codex_local_auth_subscriptions_to_plan_evidence() + .expect("migration"), + 0 + ); + + let account = store + .account(&account_id) + .expect("account") + .expect("exists"); + assert_eq!(account.plan_name, None); + assert_eq!(account.account_label.as_deref(), Some("Codex account")); + let observations = store.account_plan_observations().expect("plan evidence"); + assert_eq!(observations.len(), 1); + assert_eq!(observations[0].provider_account_id, Some(account_id)); + assert_eq!(observations[0].raw_plan_name, "future_ultra"); + assert_eq!(observations[0].plan_name, "Future Ultra"); + assert_eq!(observations[0].active_from, None); + assert_eq!(observations[0].active_until, None); + assert_eq!( + observations[0].evidence_kind, + statsai_core::AccountEvidenceKind::LegacyLocalAuth + ); + assert_eq!( + store + .migrate_legacy_codex_local_auth_subscriptions_to_plan_evidence() + .expect("repeat migration"), + 0 + ); + assert_eq!( + store + .account_plan_observations() + .expect("repeated plan evidence") + .len(), + 1 + ); + } + + #[test] + fn completed_legacy_plan_conversion_is_not_repeated_on_migrate() { + let store = Store::in_memory().expect("store"); + store + .conn + .execute( + r#" + INSERT INTO account_plan_observations ( + observation_id, provider, source_id, provider_account_id, + observed_at, active_from, active_until, plan_name, evidence_kind, payload + ) VALUES (?1, ?2, ?3, NULL, ?4, NULL, NULL, ?5, ?6, ?7) + "#, + params![ + "post-conversion-malformed-observation", + "codex", + "source-after-conversion", + "2026-08-23T00:00:00Z", + "Pro", + "quota_status", + "not-json" + ], + ) + .expect("insert evidence that must not be rescanned"); + + store.migrate().expect("completed conversion stays skipped"); + } + + #[test] + fn direct_conversation_evidence_overrides_only_that_event_and_preserves_manual_interval() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/codex-direct-account-binding"), + LocationOrigin::Configured, + ); + let observed_at = Utc + .with_ymd_and_hms(2026, 8, 1, 0, 0, 0) + .single() + .expect("observed_at"); + let manual_account = ProviderAccountId("manual-account".to_string()); + let directly_bound_account = ProviderAccountId("direct-account".to_string()); + store.upsert_source(&source).expect("source"); + let manual_assignment = SourceAccountAssignment { + schema_version: SOURCE_ACCOUNT_ASSIGNMENT_SCHEMA_VERSION.to_string(), + assignment_id: SourceAccountAssignmentId("manual-assignment".to_string()), + source_id: source.source_id.clone(), + provider: "codex".to_string(), + provider_account_id: manual_account.clone(), + started_at: observed_at - chrono::Duration::days(1), + ended_at: None, + record_source: IdentitySource::UserConfigured, + verified_at: None, + created_at: observed_at, + updated_at: observed_at, + }; + store + .upsert_source_account_assignment(&manual_assignment) + .expect("manual assignment"); + let direct_binding = statsai_core::ConversationAccountBindingV1 { + schema_version: statsai_core::CONVERSATION_ACCOUNT_BINDING_SCHEMA_VERSION.to_string(), + binding_id: "direct-binding".to_string(), + provider: "codex".to_string(), + source_id: source.source_id.clone(), + provider_account_id: directly_bound_account.clone(), + conversation_id_hash: "same-session".to_string(), + turn_id_hash: None, + observed_at, + evidence_kind: statsai_core::AccountEvidenceKind::ResetHistory, + confidence: Confidence::High, + }; + assert_eq!( + store + .upsert_conversation_account_bindings(std::slice::from_ref(&direct_binding)) + .expect("direct binding"), + 1 + ); + assert_eq!( + store + .upsert_conversation_account_bindings(&[direct_binding]) + .expect("repeat direct binding"), + 0 + ); + store + .upsert_account_identity_observations(&[statsai_core::AccountIdentityObservationV1 { + schema_version: statsai_core::ACCOUNT_IDENTITY_OBSERVATION_SCHEMA_VERSION + .to_string(), + observation_id: "direct-identity".to_string(), + provider: "codex".to_string(), + source_id: source.source_id.clone(), + provider_account_id: Some(directly_bound_account.clone()), + provider_user_id_hash: Some("provider-id-hash".to_string()), + email_hash: None, + conversation_id_hash: Some("same-session".to_string()), + turn_id_hash: None, + observed_at, + evidence_kind: statsai_core::AccountEvidenceKind::ResetHistory, + confidence: Confidence::High, + auth_mode: None, + application_version: None, + parser_version: "test.v1".to_string(), + artifact_kind: "test".to_string(), + artifact_path_hash: "path-hash".to_string(), + record_fingerprint: "record-hash".to_string(), + }]) + .expect("identity observation"); + let mut directly_bound = test_store_event(&source, observed_at, "direct"); + directly_bound.provider_account_id = Some(manual_account.clone()); + let mut unrelated = test_store_event(&source, observed_at, "unrelated"); + unrelated.session.local_session_id_hash = Some("other-session".to_string()); + unrelated.provider_account_id = Some(manual_account.clone()); + let mut events = vec![directly_bound, unrelated]; + + store + .apply_conversation_account_bindings(&source.source_id, &mut events) + .expect("apply binding"); + + assert_eq!(events[0].provider_account_id, Some(directly_bound_account)); + assert_eq!(events[1].provider_account_id, Some(manual_account.clone())); + assert_eq!( + store + .list_source_account_assignments_for_source(&source.source_id) + .expect("manual interval"), + vec![manual_assignment] + ); + let summaries = store + .account_evidence_summaries("device") + .expect("evidence summary"); + assert_eq!(summaries.len(), 1); + assert_eq!(summaries[0].directly_bound_conversations, 1); + assert_eq!(summaries[0].uncovered_gap_count, 0); + assert_eq!(summaries[0].conflict_count, 1); + } + + #[test] + fn confirmed_auth_reload_boundaries_repair_switches_conservatively() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/codex-account-evidence-switch"), + LocationOrigin::Configured, + ); + let base = Utc + .with_ymd_and_hms(2026, 8, 1, 0, 0, 0) + .single() + .expect("base"); + let account_a = ProviderAccountId("account-a".to_string()); + let account_b = ProviderAccountId("account-b".to_string()); + store.upsert_source(&source).expect("source"); + store + .upsert_source_account_assignment(&SourceAccountAssignment { + schema_version: SOURCE_ACCOUNT_ASSIGNMENT_SCHEMA_VERSION.to_string(), + assignment_id: SourceAccountAssignmentId("broad-account-a".to_string()), + source_id: source.source_id.clone(), + provider: "codex".to_string(), + provider_account_id: account_a.clone(), + started_at: base, + ended_at: None, + record_source: IdentitySource::LocalAuth, + verified_at: Some(base), + created_at: base, + updated_at: base, + }) + .expect("broad assignment"); + let observation = |id: &str, + account: ProviderAccountId, + observed_at: chrono::DateTime, + kind: statsai_core::AccountEvidenceKind| { + statsai_core::AccountIdentityObservationV1 { + schema_version: statsai_core::ACCOUNT_IDENTITY_OBSERVATION_SCHEMA_VERSION + .to_string(), + observation_id: id.to_string(), + provider: "codex".to_string(), + source_id: source.source_id.clone(), + provider_account_id: Some(account), + provider_user_id_hash: Some(format!("hash-{id}")), + email_hash: None, + conversation_id_hash: None, + turn_id_hash: None, + observed_at, + evidence_kind: kind, + confidence: Confidence::High, + auth_mode: Some("chatgpt".to_string()), + application_version: None, + parser_version: "test.v1".to_string(), + artifact_kind: "test".to_string(), + artifact_path_hash: "path-hash".to_string(), + record_fingerprint: format!("fingerprint-{id}"), + } + }; + let account_a_reload = base + chrono::Duration::days(1); + let account_a_confirmation = base + chrono::Duration::days(2); + let account_b_reload = base + chrono::Duration::days(3); + let account_b_confirmation = base + chrono::Duration::days(4); + store + .upsert_account_identity_observations(&[ + observation( + "a-reload", + account_a.clone(), + account_a_reload, + statsai_core::AccountEvidenceKind::AuthReload, + ), + observation( + "a-confirm", + account_a.clone(), + account_a_confirmation, + statsai_core::AccountEvidenceKind::TelemetryIdentity, + ), + observation( + "b-reload", + account_b.clone(), + account_b_reload, + statsai_core::AccountEvidenceKind::AuthReload, + ), + observation( + "b-confirm", + account_b.clone(), + account_b_confirmation, + statsai_core::AccountEvidenceKind::AuthSnapshot, + ), + ]) + .expect("identity evidence"); + + assert!( + store + .reconcile_source_account_evidence_assignments(&source.source_id) + .expect("repair intervals") + > 0 + ); + assert_eq!( + store + .reconcile_source_account_evidence_assignments(&source.source_id) + .expect("repeat repair"), + 0 + ); + let assignments = store + .list_source_account_assignments_for_source(&source.source_id) + .expect("assignments"); + assert_eq!(assignments.len(), 2); + let repaired_a = assignments + .iter() + .find(|assignment| assignment.provider_account_id == account_a) + .expect("account a interval"); + assert_eq!(repaired_a.started_at, base); + assert_eq!(repaired_a.ended_at, Some(account_b_reload)); + let repaired_b = assignments + .iter() + .find(|assignment| assignment.provider_account_id == account_b) + .expect("account b interval"); + assert_eq!(repaired_b.started_at, account_b_reload); + assert_eq!(repaired_b.ended_at, None); + } + #[test] fn inserts_events_idempotently() { let store = Store::in_memory().expect("store"); @@ -8680,6 +9273,8 @@ mod tests { accounts: Vec::new(), source_account_assignments: Vec::new(), subscriptions: Vec::new(), + account_plan_observations: Vec::new(), + account_evidence_summaries: Vec::new(), events: Vec::new(), summaries: summaries.clone(), task_buckets: Vec::new(), @@ -10141,6 +10736,98 @@ mod tests { ); } + #[test] + fn pending_http_sync_summary_counts_include_account_plan_and_evidence_only_uploads() { + let store = Store::in_memory().expect("store"); + let target = "https://api.example.com/api/sync/batches"; + let observed_at = Utc + .with_ymd_and_hms(2026, 8, 20, 12, 0, 0) + .single() + .expect("observed at"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/codex-pending-account-evidence"), + LocationOrigin::Configured, + ); + let account_id = ProviderAccountId("pending-account".to_string()); + store.upsert_source(&source).expect("source"); + store + .upsert_account_identity_observations(&[statsai_core::AccountIdentityObservationV1 { + schema_version: statsai_core::ACCOUNT_IDENTITY_OBSERVATION_SCHEMA_VERSION + .to_string(), + observation_id: "pending-identity".to_string(), + provider: "codex".to_string(), + source_id: source.source_id.clone(), + provider_account_id: Some(account_id.clone()), + provider_user_id_hash: Some("provider-id-hash".to_string()), + email_hash: None, + conversation_id_hash: None, + turn_id_hash: None, + observed_at, + evidence_kind: statsai_core::AccountEvidenceKind::AuthSnapshot, + confidence: Confidence::High, + auth_mode: Some("chatgpt".to_string()), + application_version: None, + parser_version: "test.v1".to_string(), + artifact_kind: "auth_json".to_string(), + artifact_path_hash: "path-hash".to_string(), + record_fingerprint: "identity-fingerprint".to_string(), + }]) + .expect("identity evidence"); + store + .upsert_account_plan_observations(&[statsai_core::AccountPlanObservationV1 { + schema_version: statsai_core::ACCOUNT_PLAN_OBSERVATION_SCHEMA_VERSION.to_string(), + observation_id: "pending-plan".to_string(), + provider: "codex".to_string(), + source_id: source.source_id, + provider_account_id: Some(account_id), + raw_plan_name: "plus".to_string(), + plan_name: "Plus".to_string(), + observed_at, + active_from: None, + active_until: None, + is_current_snapshot: true, + evidence_kind: statsai_core::AccountEvidenceKind::AuthSnapshot, + confidence: Confidence::High, + parser_version: "test.v1".to_string(), + artifact_path_hash: "path-hash".to_string(), + record_fingerprint: "plan-fingerprint".to_string(), + }]) + .expect("plan evidence"); + + let plans = store + .account_plan_projections("device") + .expect("plan projections"); + let evidence = store + .account_evidence_summaries("device") + .expect("evidence summaries"); + assert_eq!(plans.len(), 1); + assert_eq!(evidence.len(), 1); + assert_eq!( + store + .pending_http_sync_summary_counts(target, "device") + .expect("pending counts") + .total, + 2 + ); + + store + .record_account_plan_projections_synced("http", target, &plans) + .expect("record plan synced"); + store + .record_account_evidence_summaries_synced("http", target, &evidence) + .expect("record evidence synced"); + assert_eq!( + store + .pending_http_sync_summary_counts(target, "device") + .expect("settled counts") + .total, + 0 + ); + } + #[test] fn sync_preferences_round_trip_and_normalize_tasks() { let store = Store::in_memory().expect("store"); diff --git a/crates/statsai-store/src/migrations.rs b/crates/statsai-store/src/migrations.rs index 6e2d01d..e7a8876 100644 --- a/crates/statsai-store/src/migrations.rs +++ b/crates/statsai-store/src/migrations.rs @@ -2,7 +2,7 @@ use anyhow::{bail, Context, Result}; use chrono::Utc; use rusqlite::{Connection, OptionalExtension}; -pub const CURRENT_SCHEMA_VERSION: i64 = 18; +pub const CURRENT_SCHEMA_VERSION: i64 = 21; pub fn migrate(conn: &Connection) -> Result<()> { if let Some(current) = existing_schema_version(conn)? { @@ -137,6 +137,9 @@ fn apply_migration(conn: &Connection, version: i64) -> Result<()> { 16 => apply_migration_016(conn), 17 => apply_migration_017(conn), 18 => apply_migration_018(conn), + 19 => apply_migration_019(conn), + 20 => apply_migration_020(conn), + 21 => apply_migration_021(conn), _ => bail!("unsupported schema migration version {version}"), } } @@ -754,6 +757,91 @@ fn apply_migration_018(conn: &Connection) -> Result<()> { Ok(()) } +fn apply_migration_019(conn: &Connection) -> Result<()> { + conn.execute_batch( + r#" + CREATE TABLE IF NOT EXISTS account_identity_observations ( + observation_id TEXT PRIMARY KEY, + provider TEXT NOT NULL, + source_id TEXT NOT NULL, + provider_account_id TEXT, + observed_at TEXT NOT NULL, + evidence_kind TEXT NOT NULL, + conversation_id_hash TEXT, + payload TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS account_identity_observations_source_idx + ON account_identity_observations (source_id, observed_at, observation_id); + CREATE INDEX IF NOT EXISTS account_identity_observations_account_idx + ON account_identity_observations + (provider, provider_account_id, observed_at, observation_id); + + CREATE TABLE IF NOT EXISTS account_plan_observations ( + observation_id TEXT PRIMARY KEY, + provider TEXT NOT NULL, + source_id TEXT NOT NULL, + provider_account_id TEXT, + observed_at TEXT NOT NULL, + active_from TEXT, + active_until TEXT, + plan_name TEXT NOT NULL, + evidence_kind TEXT NOT NULL, + payload TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS account_plan_observations_account_idx + ON account_plan_observations + (provider, provider_account_id, observed_at, observation_id); + CREATE INDEX IF NOT EXISTS account_plan_observations_source_idx + ON account_plan_observations (source_id, observed_at, observation_id); + + CREATE TABLE IF NOT EXISTS conversation_account_bindings ( + binding_id TEXT PRIMARY KEY, + provider TEXT NOT NULL, + source_id TEXT NOT NULL, + provider_account_id TEXT NOT NULL, + conversation_id_hash TEXT NOT NULL, + turn_id_hash TEXT, + observed_at TEXT NOT NULL, + payload TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS conversation_account_bindings_lookup_idx + ON conversation_account_bindings + (source_id, conversation_id_hash, turn_id_hash, observed_at); + "#, + )?; + Ok(()) +} + +fn apply_migration_020(conn: &Connection) -> Result<()> { + conn.execute_batch( + r#" + CREATE TABLE IF NOT EXISTS account_evidence_checkpoints ( + source_id TEXT NOT NULL, + artifact_path_hash TEXT NOT NULL, + parser_version TEXT NOT NULL, + maximum_row_id INTEGER NOT NULL, + database_size INTEGER NOT NULL, + database_modified_nanos INTEGER NOT NULL, + wal_size INTEGER NOT NULL, + wal_modified_nanos INTEGER NOT NULL, + payload TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (source_id, artifact_path_hash, parser_version) + ); + CREATE INDEX IF NOT EXISTS account_evidence_checkpoints_source_idx + ON account_evidence_checkpoints (source_id, artifact_path_hash, parser_version); + "#, + )?; + Ok(()) +} + +fn apply_migration_021(conn: &Connection) -> Result<()> { + conn.execute_batch( + "ALTER TABLE account_evidence_checkpoints ADD COLUMN checkpoint_row_fingerprint TEXT;", + )?; + Ok(()) +} + fn ensure_local_task_tables(conn: &Connection) -> Result<()> { conn.execute_batch( r#" @@ -914,6 +1002,15 @@ mod tests { assert!(table_exists(&conn, "code_git_identities")); assert!(table_exists(&conn, "code_change_matches")); assert!(table_exists(&conn, "code_change_metrics")); + assert!(table_exists(&conn, "account_identity_observations")); + assert!(table_exists(&conn, "account_plan_observations")); + assert!(table_exists(&conn, "conversation_account_bindings")); + assert!(table_exists(&conn, "account_evidence_checkpoints")); + assert!(column_exists( + &conn, + "account_evidence_checkpoints", + "checkpoint_row_fingerprint" + )); } #[test] @@ -926,20 +1023,20 @@ mod tests { applied_at TEXT NOT NULL ); INSERT INTO schema_migrations (version, applied_at) - VALUES (19, '2026-08-19T00:00:00Z'); + VALUES (22, '2026-08-23T00:00:00Z'); "#, ) .expect("create future schema marker"); - let error = migrate(&conn).expect_err("schema 19 must be rejected by schema 18 binary"); + let error = migrate(&conn).expect_err("schema 22 must be rejected by schema 21 binary"); assert_eq!( error.to_string(), - "database schema version 19 is newer than this StatsAI binary supports (18); upgrade StatsAI or use a compatible database" + "database schema version 22 is newer than this StatsAI binary supports (21); upgrade StatsAI or use a compatible database" ); assert_eq!( current_schema_version(&conn).expect("read unchanged version"), - 19 + 22 ); } diff --git a/crates/statsai-store/src/quota.rs b/crates/statsai-store/src/quota.rs index a615822..8cd092b 100644 --- a/crates/statsai-store/src/quota.rs +++ b/crates/statsai-store/src/quota.rs @@ -439,7 +439,7 @@ impl Store { }) } - fn quota_observations_for_source( + pub(crate) fn quota_observations_for_source( &self, source_id: &SourceId, ) -> Result> { @@ -1685,6 +1685,100 @@ mod tests { (source.source_id, account_id) } + #[test] + fn quota_plan_evidence_rebuild_replaces_corrected_plan_rows() { + let store = Store::in_memory().expect("store"); + let observed_at = DateTime::from_timestamp(1_787_227_200, 0).expect("observed at"); + let reset_at = observed_at + Duration::days(7); + let (source_id, account_id) = assigned_source(&store, observed_at - Duration::days(1)); + let mut record = sample_record( + source_id.clone(), + "quota-plan", + "quota-plan-v1", + observed_at, + reset_at.timestamp(), + "primary", + 10_080, + 25.0, + ); + store + .upsert_quota_observations(std::slice::from_ref(&record)) + .expect("initial quota row"); + store + .rebuild_quota_plan_observations_for_source(&source_id) + .expect("initial plan rebuild"); + let initial = store.account_plan_observations().expect("initial plan"); + assert_eq!(initial.len(), 1); + assert_eq!(initial[0].raw_plan_name, "pro"); + assert_eq!(initial[0].provider_account_id, Some(account_id.clone())); + + record.observation.status.plan_type = Some("future_ultra".to_string()); + record.observation.semantic_fingerprint = "quota-plan-v2".to_string(); + store + .upsert_quota_observations(&[record]) + .expect("corrected quota row"); + store + .rebuild_quota_plan_observations_for_source(&source_id) + .expect("corrected plan rebuild"); + + let corrected = store.account_plan_observations().expect("corrected plan"); + assert_eq!(corrected.len(), 1); + assert_eq!(corrected[0].raw_plan_name, "future_ultra"); + assert_eq!(corrected[0].plan_name, "Future Ultra"); + assert_eq!(corrected[0].provider_account_id, Some(account_id)); + assert_ne!(corrected[0].observation_id, initial[0].observation_id); + } + + #[test] + fn quota_plan_label_never_identifies_an_unassigned_account() { + let store = Store::in_memory().expect("store"); + let observed_at = DateTime::from_timestamp(1_787_227_200, 0).expect("observed_at"); + let reset_at = observed_at.timestamp() + 3_600; + let unassigned = sample_record( + SourceId("unassigned-source".to_string()), + "quota-plan-unassigned", + "quota-plan-unassigned-fingerprint", + observed_at, + reset_at, + "primary", + 300, + 25.0, + ); + + assert_eq!( + store + .upsert_quota_plan_observations(&[unassigned]) + .expect("unassigned plan label"), + 0 + ); + assert!(store + .account_plan_observations() + .expect("no plan observations") + .is_empty()); + + let (source_id, account_id) = assigned_source(&store, observed_at); + let assigned = sample_record( + source_id, + "quota-plan-assigned", + "quota-plan-assigned-fingerprint", + observed_at, + reset_at, + "primary", + 300, + 25.0, + ); + assert_eq!( + store + .upsert_quota_plan_observations(&[assigned]) + .expect("assigned plan label"), + 1 + ); + let observations = store.account_plan_observations().expect("plan observation"); + assert_eq!(observations.len(), 1); + assert_eq!(observations[0].provider_account_id, Some(account_id)); + assert_eq!(observations[0].raw_plan_name, "pro"); + } + #[test] fn semantic_collapse_prefers_attributed_linked_evidence() { let now = Utc::now(); diff --git a/crates/statsai-store/tests/forward_schema.rs b/crates/statsai-store/tests/forward_schema.rs index cbad9af..3867f84 100644 --- a/crates/statsai-store/tests/forward_schema.rs +++ b/crates/statsai-store/tests/forward_schema.rs @@ -6,6 +6,7 @@ fn store_refuses_to_open_database_from_a_newer_binary() { let directory = tempfile::tempdir().expect("temporary directory"); let path = directory.path().join("future.sqlite"); let connection = Connection::open(&path).expect("create future database"); + let future_schema_version = CURRENT_SCHEMA_VERSION + 1; connection .execute_batch( r#" @@ -13,22 +14,29 @@ fn store_refuses_to_open_database_from_a_newer_binary() { version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL ); - INSERT INTO schema_migrations (version, applied_at) - VALUES (19, '2026-08-19T00:00:00Z'); "#, ) + .expect("create migration table"); + connection + .execute( + "INSERT INTO schema_migrations (version, applied_at) VALUES (?1, ?2)", + (future_schema_version, "2026-08-23T00:00:00Z"), + ) .expect("record future schema"); drop(connection); let error = match Store::open(&path) { - Ok(_) => panic!("schema 18 binary must reject schema 19 database"), + Ok(_) => panic!( + "schema {CURRENT_SCHEMA_VERSION} binary must reject schema {future_schema_version} database" + ), Err(error) => error, }; - assert_eq!(CURRENT_SCHEMA_VERSION, 18); assert!(error .to_string() - .contains("database schema version 19 is newer than this StatsAI binary supports (18)")); + .contains(&format!( + "database schema version {future_schema_version} is newer than this StatsAI binary supports ({CURRENT_SCHEMA_VERSION})" + ))); let connection = Connection::open(&path).expect("reopen rejected database"); assert_eq!( connection @@ -36,6 +44,6 @@ fn store_refuses_to_open_database_from_a_newer_binary() { row.get::<_, i64>(0) }) .expect("read future schema marker"), - 19 + future_schema_version ); } diff --git a/crates/statsai-sync/src/lib.rs b/crates/statsai-sync/src/lib.rs index ad981f1..6e631dd 100644 --- a/crates/statsai-sync/src/lib.rs +++ b/crates/statsai-sync/src/lib.rs @@ -3,8 +3,9 @@ use anyhow::{bail, Context, Result}; use statsai_core::{ SyncAck, SyncBatch, SYNC_ACK_V1_SCHEMA_VERSION, SYNC_ACK_V2_SCHEMA_VERSION, - SYNC_ACK_V3_SCHEMA_VERSION, SYNC_ACK_V4_SCHEMA_VERSION, SYNC_BATCH_V1_SCHEMA_VERSION, - SYNC_BATCH_V2_SCHEMA_VERSION, SYNC_BATCH_V3_SCHEMA_VERSION, SYNC_BATCH_V4_SCHEMA_VERSION, + SYNC_ACK_V3_SCHEMA_VERSION, SYNC_ACK_V4_SCHEMA_VERSION, SYNC_ACK_V5_SCHEMA_VERSION, + SYNC_BATCH_V1_SCHEMA_VERSION, SYNC_BATCH_V2_SCHEMA_VERSION, SYNC_BATCH_V3_SCHEMA_VERSION, + SYNC_BATCH_V4_SCHEMA_VERSION, SYNC_BATCH_V5_SCHEMA_VERSION, }; use std::io::Write; use std::path::{Path, PathBuf}; @@ -180,6 +181,7 @@ fn validate_sync_ack(batch: &SyncBatch, ack: &SyncAck) -> Result<()> { SYNC_BATCH_V2_SCHEMA_VERSION => SYNC_ACK_V2_SCHEMA_VERSION, SYNC_BATCH_V3_SCHEMA_VERSION => SYNC_ACK_V3_SCHEMA_VERSION, SYNC_BATCH_V4_SCHEMA_VERSION => SYNC_ACK_V4_SCHEMA_VERSION, + SYNC_BATCH_V5_SCHEMA_VERSION => SYNC_ACK_V5_SCHEMA_VERSION, other => bail!("unsupported sync batch schema {other}"), }; if ack.schema_version != expected_ack_schema { @@ -232,6 +234,24 @@ fn validate_sync_ack(batch: &SyncBatch, ack: &SyncAck) -> Result<()> { ack.accepted.subscriptions, ack.duplicates.subscriptions, )?; + validate_sync_ack_counts( + "account_plan_observations", + batch.account_plan_observations.len() as u64, + ack.accepted.account_plan_observations, + ack.duplicates.account_plan_observations, + )?; + validate_sync_ack_counts( + "account_evidence_summaries", + batch.account_evidence_summaries.len() as u64, + ack.accepted.account_evidence_summaries, + ack.duplicates.account_evidence_summaries, + )?; + validate_sync_ack_counts( + "quota_cycle_contributions", + batch.quota_cycle_contributions.len() as u64, + ack.accepted.quota_cycle_contributions, + ack.duplicates.quota_cycle_contributions, + )?; validate_sync_ack_counts( "events", batch.events.len() as u64, @@ -268,6 +288,18 @@ fn validate_sync_ack(batch: &SyncBatch, ack: &SyncAck) -> Result<()> { ack.accepted.quota_cycle_contributions, ack.duplicates.quota_cycle_contributions, )?; + validate_sync_ack_counts( + "account_plan_observations", + batch.account_plan_observations.len() as u64, + ack.accepted.account_plan_observations, + ack.duplicates.account_plan_observations, + )?; + validate_sync_ack_counts( + "account_evidence_summaries", + batch.account_evidence_summaries.len() as u64, + ack.accepted.account_evidence_summaries, + ack.duplicates.account_evidence_summaries, + )?; Ok(()) } @@ -386,6 +418,8 @@ mod tests { accounts: Vec::new(), source_account_assignments: Vec::new(), subscriptions: Vec::new(), + account_plan_observations: Vec::new(), + account_evidence_summaries: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: Vec::new(), @@ -493,7 +527,7 @@ mod tests { let (auth, content_type, body) = rx.recv().expect("request body"); assert_eq!(auth.as_deref(), Some("Bearer token_123")); assert_eq!(content_type.as_deref(), Some("application/json")); - assert!(body.contains("\"schema_version\":\"sync_batch.v4\"")); + assert!(body.contains("\"schema_version\":\"sync_batch.v5\"")); assert!(body.contains("\"batch_id\":\"batch_1\"")); } @@ -592,8 +626,8 @@ mod tests { )) .expect("v1 ack"); - let error = validate_sync_ack(&batch, &ack).expect_err("v4 batch with v1 ack"); - assert!(error.to_string().contains("requires sync_ack.v4")); + let error = validate_sync_ack(&batch, &ack).expect_err("v5 batch with v1 ack"); + assert!(error.to_string().contains("requires sync_ack.v5")); batch.schema_version = SYNC_BATCH_V1_SCHEMA_VERSION.to_string(); ack.schema_version = SYNC_ACK_V2_SCHEMA_VERSION.to_string(); @@ -610,6 +644,10 @@ mod tests { batch.schema_version = SYNC_BATCH_V4_SCHEMA_VERSION.to_string(); ack.schema_version = SYNC_ACK_V4_SCHEMA_VERSION.to_string(); validate_sync_ack(&batch, &ack).expect("matching v4 schemas"); + + batch.schema_version = SYNC_BATCH_V5_SCHEMA_VERSION.to_string(); + ack.schema_version = SYNC_ACK_V5_SCHEMA_VERSION.to_string(); + validate_sync_ack(&batch, &ack).expect("matching v5 schemas"); } #[test] diff --git a/crates/statsai/src/main.rs b/crates/statsai/src/main.rs index 26bdb2f..018fac5 100644 --- a/crates/statsai/src/main.rs +++ b/crates/statsai/src/main.rs @@ -6,16 +6,23 @@ use serde_json::{json, Value}; #[cfg(test)] use statsai_adapters::VerifiedSubscriptionState; use statsai_adapters::{ - adapter_for_provider, default_adapters, ArchiveScan, ProviderAdapter, ScanCandidateFile, - ScanDiagnostics, ScanOptions, VerifiedSourceObservation, + adapter_for_provider, default_adapters, remap_account_evidence_account_ids, + retain_accounts_referenced_by_account_evidence, AccountEvidenceScan, ArchiveScan, + ProviderAdapter, ScanCandidateFile, ScanDiagnostics, ScanOptions, VerifiedSourceObservation, }; #[cfg(test)] use statsai_adapters::{SourceIdentityInference, VerifiedSourceState}; +#[cfg(test)] +use statsai_core::{ + account_plan_observation_id, conversation_account_binding_id, micro_usd_to_cents_rounded, + provider_account_id, summary_id, Confidence, CostInfo, EventSource, PrivacyInfo, PrivacyMode, + SummaryMetadata, UsageCounts, USAGE_SUMMARY_SCHEMA_VERSION, +}; use statsai_core::{ build_usage_report, display_account_identity, expand_home_path, hash_text, home_dir, normalize_email, normalize_provider_user_id, path_hash, periods_overlap, - project_contains_file_paths, project_has_stable_identity, report_period_from_range, - sanitize_code_change_metric_for_sync, sanitize_task_bucket_for_sync, + project_contains_file_paths, project_has_stable_identity, provider_account_id_from_identity, + report_period_from_range, sanitize_code_change_metric_for_sync, sanitize_task_bucket_for_sync, source_account_assignment_id, source_id as statsai_source_id, subscription_id, timestamp_in_period, ArchiveContentKind, ArchiveConversation, BillingPeriod, EventId, IdentitySource, LocationOrigin, ProjectInfo, ProviderAccount, ProviderAccountId, @@ -27,12 +34,6 @@ use statsai_core::{ UsageSummary, UsageTotals, WorkItem, WorkItemId, SOURCE_ACCOUNT_ASSIGNMENT_SCHEMA_VERSION, SUBSCRIPTION_SCHEMA_VERSION, SYNC_BATCH_SCHEMA_VERSION, }; -#[cfg(test)] -use statsai_core::{ - micro_usd_to_cents_rounded, provider_account_id, provider_account_id_from_identity, summary_id, - Confidence, CostInfo, EventSource, PrivacyInfo, PrivacyMode, SummaryMetadata, UsageCounts, - USAGE_SUMMARY_SCHEMA_VERSION, -}; use statsai_sdk::{ build_reported_usage_summary, ReportedUsageSummaryInput, ReportedUsageSummaryRecord, REPORTED_USAGE_IMPORT_ADAPTER_ID, @@ -1088,6 +1089,36 @@ fn scan_with_adapters( if source.path_label.is_none() { source.path_label = path_label_from_hashless_source(&source); } + let verification_mode = source_verification_mode(&source); + let account_evidence_enabled = + matches!(verification_mode, SourceVerificationMode::Auto); + let mut account_evidence = AccountEvidenceScan::default(); + if account_evidence_enabled { + let account_evidence_checkpoints = + store.account_evidence_checkpoints(&source.source_id)?; + account_evidence = + adapter.collect_account_evidence(&source, &account_evidence_checkpoints)?; + let known_account_aliases = canonicalize_known_account_evidence( + store, + adapter.provider(), + &mut account_evidence, + )?; + store.retain_unseen_account_evidence( + &source.source_id, + &mut account_evidence.identity_observations, + &mut account_evidence.plan_observations, + &mut account_evidence.conversation_bindings, + )?; + retain_accounts_referenced_by_account_evidence( + adapter.provider(), + &known_account_aliases, + &mut account_evidence, + ); + } + let account_evidence_count = account_evidence.identity_observations.len() + + account_evidence.plan_observations.len() + + account_evidence.conversation_bindings.len(); + let account_evidence_checkpoint_count = account_evidence.checkpoints.len(); let cache_candidates = adapter.scan_candidates(&source)?; let compatible_scan_signatures = scan_candidate_compatible_signatures(&cache_candidates); @@ -1139,7 +1170,6 @@ fn scan_with_adapters( .collect() }), }; - let verification_mode = source_verification_mode(&source); let probed_verified_source_state = if matches!(verification_mode, SourceVerificationMode::Disabled) { VerifiedSourceObservation::Unavailable @@ -1216,6 +1246,8 @@ fn scan_with_adapters( || scan.diagnostics.files_scanned > 0 || scan.diagnostics.files_skipped_unchanged > 0 || log_rows > 0 + || account_evidence_count > 0 + || account_evidence_checkpoint_count > 0 || verified_state_changed; let suppress_source_processing = !command.verbose && !command.explain @@ -1225,6 +1257,8 @@ fn scan_with_adapters( && source_quota_observation_count == 0 && !touched_files && !has_cache_entry_upgrades + && account_evidence_count == 0 + && account_evidence_checkpoint_count == 0 && !verified_state_changed; if !has_scan_activity { @@ -1282,12 +1316,32 @@ fn scan_with_adapters( next_verified_state_hash, )?; persist_source_after_preview(store, &source)?; + if account_evidence_enabled { + canonicalize_account_evidence( + store, + adapter.provider(), + &mut account_evidence, + )?; + store.upsert_account_identity_observations( + &account_evidence.identity_observations, + )?; + store.upsert_account_plan_observations(&account_evidence.plan_observations)?; + store.reconcile_source_account_evidence_assignments(&source.source_id)?; + store.upsert_conversation_account_bindings( + &account_evidence.conversation_bindings, + )?; + store.reattribute_conversation_bound_events(&source.source_id)?; + } apply_source_account_resolution( store, &source, &mut scan.events, &mut scan.summaries, )?; + if account_evidence_enabled { + store + .apply_conversation_account_bindings(&source.source_id, &mut scan.events)?; + } let mut affected_project_buckets = if command.include_tasks { scan.task_spans .iter() @@ -1355,6 +1409,7 @@ fn scan_with_adapters( &insert_result.canonical_event_ids, ); store.upsert_quota_observations(&scan.quota_observations)?; + store.rebuild_quota_plan_observations_for_source(&source.source_id)?; store.clear_orphaned_quota_usage_links()?; if command.include_tasks { rewrite_task_span_linked_event_ids( @@ -1403,6 +1458,9 @@ fn scan_with_adapters( .upgrade_scan_file_entries(&source.source_id, &compatible_entries_to_upgrade)?; let removed_cache_keys = scan_file_cache_keys(&removed_file_entries); store.delete_scan_file_entries(&source.source_id, &removed_cache_keys)?; + if account_evidence_enabled { + store.upsert_account_evidence_checkpoints(&account_evidence.checkpoints)?; + } if command.include_tasks && !rebuild_project_buckets.is_empty() @@ -1742,6 +1800,11 @@ fn source(command: SourceCommand, store: &Store, device_id: &str) -> Result<()> } else { 0 }; + let deleted_account_evidence = if delete_data { + store.delete_account_evidence_for_sources(std::slice::from_ref(&source_id))? + } else { + 0 + }; if delete_data { store.clear_orphaned_quota_usage_links()?; } @@ -1782,6 +1845,7 @@ fn source(command: SourceCommand, store: &Store, device_id: &str) -> Result<()> "deleted_scan_cache_entries": deleted_scan_entries, "deleted_task_spans": deleted_task_spans.deleted, "deleted_quota_observations": deleted_quota_observations, + "deleted_account_evidence": deleted_account_evidence, "deleted_code_change_traces": deleted_trace_edits, "work_items_rebuilt": rebuilt_work_items, "code_change_metrics_rebuilt": rebuilt_code_change_metrics, @@ -1841,6 +1905,10 @@ fn source(command: SourceCommand, store: &Store, device_id: &str) -> Result<()> if matches!(source.verification_mode, SourceVerificationMode::Disabled) { close_active_verified_source_linkages(store, &source.source_id, Utc::now())?; } + if !matches!(source.verification_mode, SourceVerificationMode::Auto) { + store + .delete_account_evidence_for_sources(std::slice::from_ref(&source.source_id))?; + } source.updated_at = Utc::now(); store.upsert_source(&source)?; println!("{}", serde_json::to_string_pretty(&source)?); @@ -2590,6 +2658,9 @@ struct AccountReferenceCounts { events: usize, summaries: usize, quota_observations: usize, + identity_observations: usize, + plan_observations: usize, + conversation_bindings: usize, } impl AccountReferenceCounts { @@ -2599,6 +2670,9 @@ impl AccountReferenceCounts { + self.events + self.summaries + self.quota_observations + + self.identity_observations + + self.plan_observations + + self.conversation_bindings } } @@ -2613,6 +2687,9 @@ struct AccountMergeReport { moved_subscriptions: usize, moved_events: usize, moved_summaries: usize, + moved_identity_observations: usize, + moved_plan_observations: usize, + moved_conversation_bindings: usize, deleted_source_account: bool, remaining_references: AccountReferenceCounts, reset_local_sync_tracking: bool, @@ -2667,6 +2744,8 @@ fn merge_provider_accounts( .filter(|summary| summary.provider == provider) .filter(|summary| summary.provider_account_id.as_ref() == Some(&from.provider_account_id)) .count(); + let evidence_to_move = + store.account_evidence_reference_counts(provider, &from.provider_account_id)?; if !dry_run { for assignment in &assignments_to_move { @@ -2715,6 +2794,9 @@ fn merge_provider_accounts( moved_subscriptions: subscriptions_to_move.len(), moved_events: direct_events_to_move, moved_summaries: direct_summaries_to_move, + moved_identity_observations: evidence_to_move.identity_observations, + moved_plan_observations: evidence_to_move.plan_observations, + moved_conversation_bindings: evidence_to_move.conversation_bindings, deleted_source_account, remaining_references, reset_local_sync_tracking: !dry_run, @@ -2733,13 +2815,16 @@ fn remove_orphan_provider_account( account_reference_counts(store, &account.provider_account_id, Some(provider))?; if remaining_references.total() > 0 { bail!( - "account {} still has references: {} source assignments, {} subscriptions, {} events, {} summaries, {} quota observations", + "account {} still has references: {} source assignments, {} subscriptions, {} events, {} summaries, {} quota observations, {} identity observations, {} plan observations, {} conversation bindings", display_account_identity(&account), remaining_references.source_account_assignments, remaining_references.subscriptions, remaining_references.events, remaining_references.summaries, - remaining_references.quota_observations + remaining_references.quota_observations, + remaining_references.identity_observations, + remaining_references.plan_observations, + remaining_references.conversation_bindings ); } let deleted = if dry_run { @@ -2902,6 +2987,12 @@ fn move_direct_account_records( store.rewrite_summaries(&summaries_to_move)?; } + store.rekey_account_evidence( + provider, + from_provider_account_id, + target_provider_account_id, + )?; + Ok(()) } @@ -2944,6 +3035,34 @@ fn account_reference_counts( }) .filter(|record| provider_matches(&record.observation.provider)) .count(); + let evidence = if let Some(provider) = provider { + store.account_evidence_reference_counts(provider, provider_account_id)? + } else { + let identity_observations = store + .account_identity_observations(None)? + .into_iter() + .filter(|observation| { + observation.provider_account_id.as_ref() == Some(provider_account_id) + }) + .count(); + let plan_observations = store + .account_plan_observations()? + .into_iter() + .filter(|observation| { + observation.provider_account_id.as_ref() == Some(provider_account_id) + }) + .count(); + let conversation_bindings = store + .conversation_account_bindings(None)? + .into_iter() + .filter(|binding| binding.provider_account_id == *provider_account_id) + .count(); + statsai_store::AccountEvidenceReferenceCounts { + identity_observations, + plan_observations, + conversation_bindings, + } + }; Ok(AccountReferenceCounts { source_account_assignments, @@ -2951,6 +3070,9 @@ fn account_reference_counts( events, summaries, quota_observations, + identity_observations: evidence.identity_observations, + plan_observations: evidence.plan_observations, + conversation_bindings: evidence.conversation_bindings, }) } @@ -4044,8 +4166,11 @@ fn build_sync_batch_with_identity_key( let all_subscriptions: Vec<_> = store .list_subscriptions()? .into_iter() + .filter(|subscription| subscription.record_source != IdentitySource::LocalAuth) .map(sanitize_subscription_for_sync) .collect(); + let all_account_plan_observations = store.account_plan_projections(device_id)?; + let all_account_evidence_summaries = store.account_evidence_summaries(device_id)?; let snapshot_source_ids = all_sources .iter() .map(|source| source.source_id.clone()) @@ -4062,6 +4187,14 @@ fn build_sync_batch_with_identity_key( .iter() .map(|subscription| subscription.subscription_id.clone()) .collect::>(); + let snapshot_account_plan_observation_ids = all_account_plan_observations + .iter() + .map(|observation| observation.projection_id.clone()) + .collect::>(); + let snapshot_account_evidence_summary_ids = all_account_evidence_summaries + .iter() + .map(|summary| summary.summary_id.clone()) + .collect::>(); let mut authoritative_snapshot = None; let sources = if payload_mode == SyncPayloadMode::Rollups { store.pending_sources_for_sync(&command.sink, target, &all_sources)? @@ -4087,6 +4220,24 @@ fn build_sync_batch_with_identity_key( } else { all_subscriptions }; + let account_plan_observations = if payload_mode == SyncPayloadMode::Rollups { + store.pending_account_plan_projections_for_sync( + &command.sink, + target, + &all_account_plan_observations, + )? + } else { + all_account_plan_observations + }; + let account_evidence_summaries = if payload_mode == SyncPayloadMode::Rollups { + store.pending_account_evidence_summaries_for_sync( + &command.sink, + target, + &all_account_evidence_summaries, + )? + } else { + all_account_evidence_summaries + }; let (task_buckets, task_verifications) = if sync_preferences.include_tasks { let task_verification_cursor = if command.sink == "http" || command.since_last { store.sync_task_verification_cursor(&command.sink, target)? @@ -4195,6 +4346,8 @@ fn build_sync_batch_with_identity_key( .collect(), code_change_metric_ids: all_code_change_metric_ids, quota_cycle_contribution_ids: all_quota_cycle_contribution_ids, + account_plan_observation_ids: snapshot_account_plan_observation_ids, + account_evidence_summary_ids: snapshot_account_evidence_summary_ids, }; let failed_without_resume = state.as_ref().is_some_and(|state| { state.failure_count > 0 && state.pending_resume_batch_id.is_none() @@ -4252,6 +4405,8 @@ fn build_sync_batch_with_identity_key( accounts, source_account_assignments, subscriptions, + account_plan_observations, + account_evidence_summaries, events, summaries, task_buckets, @@ -4301,6 +4456,12 @@ fn record_rollup_sync_chunk_success( target, &batch.quota_cycle_contributions, )?; + store.record_account_plan_projections_synced(sink, target, &batch.account_plan_observations)?; + store.record_account_evidence_summaries_synced( + sink, + target, + &batch.account_evidence_summaries, + )?; snapshot::invalidate_dashboard_cache(); Ok(()) } @@ -4328,6 +4489,17 @@ fn record_sync_batch_success( )?; store.record_task_verifications_synced(sink, target, &batch.task_verifications)?; store.record_code_change_metrics_synced(sink, target, &batch.code_change_metrics)?; + store.record_account_plan_projections_synced(sink, target, &batch.account_plan_observations)?; + store.record_account_evidence_summaries_synced( + sink, + target, + &batch.account_evidence_summaries, + )?; + store.record_quota_cycle_contributions_synced( + sink, + target, + &batch.quota_cycle_contributions, + )?; store.mark_code_change_metrics_synced( &batch .code_change_metrics @@ -4335,11 +4507,6 @@ fn record_sync_batch_success( .map(|metric| metric.metric_id.clone()) .collect::>(), )?; - store.record_quota_cycle_contributions_synced( - sink, - target, - &batch.quota_cycle_contributions, - )?; snapshot::invalidate_dashboard_cache(); Ok(()) } @@ -4655,6 +4822,8 @@ fn should_retry_http_rollup_chunk_after_error(chunk: &SyncBatch, error: &anyhow: || chunk.accounts.len() > 1 || chunk.source_account_assignments.len() > 1 || chunk.subscriptions.len() > 1 + || chunk.account_plan_observations.len() > 1 + || chunk.account_evidence_summaries.len() > 1 || chunk.task_buckets.len() > 1 || chunk.task_verifications.len() > 1 || chunk.code_change_metrics.len() > 1 @@ -4739,6 +4908,8 @@ fn split_authoritative_snapshot( provider_account_ids: Vec::new(), source_account_assignment_ids: Vec::new(), subscription_ids: Vec::new(), + account_plan_observation_ids: Vec::new(), + account_evidence_summary_ids: Vec::new(), summary_ids: Vec::new(), code_change_metric_ids: Vec::new(), quota_cycle_contribution_ids: Vec::new(), @@ -4764,6 +4935,14 @@ fn split_authoritative_snapshot( source_account_assignment_ids ); append_ids!(snapshot.subscription_ids, subscription_ids); + append_ids!( + snapshot.account_plan_observation_ids, + account_plan_observation_ids + ); + append_ids!( + snapshot.account_evidence_summary_ids, + account_evidence_summary_ids + ); append_ids!(snapshot.summary_ids, summary_ids); append_ids!(snapshot.code_change_metric_ids, code_change_metric_ids); append_ids!( @@ -4786,6 +4965,8 @@ fn authoritative_snapshot_id_count(snapshot: &SyncAuthoritativeSnapshot) -> usiz + snapshot.provider_account_ids.len() + snapshot.source_account_assignment_ids.len() + snapshot.subscription_ids.len() + + snapshot.account_plan_observation_ids.len() + + snapshot.account_evidence_summary_ids.len() + snapshot.summary_ids.len() + snapshot.code_change_metric_ids.len() + snapshot.quota_cycle_contribution_ids.len() @@ -4946,6 +5127,24 @@ fn split_http_rollup_sync_batch_after_budget_error(batch: &SyncBatch) -> Vec 1 { return split_http_rollup_metadata_chunks(batch, batch.subscriptions.len().div_ceil(2)); } + if batch.account_plan_observations.len() > 1 { + return split_http_rollup_metadata_chunks( + batch, + batch.account_plan_observations.len().div_ceil(2), + ); + } + if batch.account_evidence_summaries.len() > 1 { + return split_http_rollup_metadata_chunks( + batch, + batch.account_evidence_summaries.len().div_ceil(2), + ); + } + if batch.quota_cycle_contributions.len() > 1 { + return split_http_rollup_metadata_chunks( + batch, + batch.quota_cycle_contributions.len().div_ceil(2), + ); + } vec![batch.clone()] } @@ -4972,6 +5171,9 @@ fn http_rollup_metadata_count(batch: &SyncBatch) -> usize { + batch.accounts.len() + batch.source_account_assignments.len() + batch.subscriptions.len() + + batch.account_plan_observations.len() + + batch.account_evidence_summaries.len() + + batch.quota_cycle_contributions.len() } fn split_http_rollup_task_chunks( @@ -5090,6 +5292,21 @@ fn estimate_http_rollup_d1_queries(batch: &SyncBatch) -> usize { })) .len(), HTTP_ROLLUP_D1_QUERY_CHUNK_SIZE, + ) + http_rollup_query_chunks( + unique_non_empty_provider_account_ids( + batch + .account_plan_observations + .iter() + .map(|observation| observation.provider_account_id.0.as_str()) + .chain( + batch + .account_evidence_summaries + .iter() + .map(|summary| summary.provider_account_id.0.as_str()), + ), + ) + .len(), + HTTP_ROLLUP_D1_QUERY_CHUNK_SIZE, ); let existing_summary_state_queries = http_rollup_query_chunks(batch.summaries.len(), HTTP_ROLLUP_D1_QUERY_CHUNK_SIZE); @@ -5100,7 +5317,10 @@ fn estimate_http_rollup_d1_queries(batch: &SyncBatch) -> usize { let metadata_write_queries = batch.sources.len() + batch.accounts.len() + batch.source_account_assignments.len() - + batch.subscriptions.len(); + + batch.subscriptions.len() + + batch.account_plan_observations.len() + + batch.account_evidence_summaries.len() + + batch.quota_cycle_contributions.len(); let project_write_queries = http_rollup_project_count(batch) + http_rollup_project_location_count(batch); let daily_rollup_write_queries = http_rollup_query_chunks( @@ -5434,6 +5654,21 @@ fn split_http_rollup_metadata_chunks(batch: &SyncBatch, chunk_size: usize) -> Ve "subscriptions", chunk_size, )); + chunks.extend(split_http_rollup_single_metadata_kind( + batch, + "account_plans", + chunk_size, + )); + chunks.extend(split_http_rollup_single_metadata_kind( + batch, + "account_evidence", + chunk_size, + )); + chunks.extend(split_http_rollup_single_metadata_kind( + batch, + "quota_cycles", + chunk_size, + )); chunks } @@ -5486,6 +5721,39 @@ fn split_http_rollup_single_metadata_kind( chunk }) .collect(), + "account_plans" => batch + .account_plan_observations + .chunks(chunk_size) + .enumerate() + .map(|(index, records)| { + let mut chunk = + empty_http_rollup_chunk(batch, &format!("account_plans_{}", index + 1)); + chunk.account_plan_observations = records.to_vec(); + chunk + }) + .collect(), + "account_evidence" => batch + .account_evidence_summaries + .chunks(chunk_size) + .enumerate() + .map(|(index, records)| { + let mut chunk = + empty_http_rollup_chunk(batch, &format!("account_evidence_{}", index + 1)); + chunk.account_evidence_summaries = records.to_vec(); + chunk + }) + .collect(), + "quota_cycles" => batch + .quota_cycle_contributions + .chunks(chunk_size) + .enumerate() + .map(|(index, records)| { + let mut chunk = + empty_http_rollup_chunk(batch, &format!("quota_cycles_{}", index + 1)); + chunk.quota_cycle_contributions = records.to_vec(); + chunk + }) + .collect(), _ => Vec::new(), } } @@ -5504,6 +5772,8 @@ fn split_http_rollup_summary_chunks(batch: &SyncBatch, chunk_size: usize) -> Vec chunk.accounts.clear(); chunk.source_account_assignments.clear(); chunk.subscriptions.clear(); + chunk.account_plan_observations.clear(); + chunk.account_evidence_summaries.clear(); chunk.events.clear(); chunk.summaries = summaries.to_vec(); chunk.task_buckets.clear(); @@ -5523,6 +5793,8 @@ fn empty_http_rollup_chunk(batch: &SyncBatch, suffix: &str) -> SyncBatch { chunk.accounts.clear(); chunk.source_account_assignments.clear(); chunk.subscriptions.clear(); + chunk.account_plan_observations.clear(); + chunk.account_evidence_summaries.clear(); chunk.events.clear(); chunk.summaries.clear(); chunk.task_buckets.clear(); @@ -8657,9 +8929,71 @@ fn reattribute_source_records(store: &Store, source_id: &SourceId) -> Result<()> store.rewrite_events(&events)?; store.rewrite_summaries(&summaries)?; store.reattribute_quota_observations(source_id)?; + store.rebuild_quota_plan_observations_for_source(source_id)?; Ok(()) } +fn canonicalize_account_evidence( + store: &Store, + provider: &str, + evidence: &mut AccountEvidenceScan, +) -> Result<()> { + let mut canonical_ids = HashMap::new(); + for observed in &evidence.accounts { + let Some(detected_id) = provider_account_id_from_identity( + provider, + observed.provider_user_id.as_deref(), + observed.email.as_deref(), + ) else { + continue; + }; + let account = upsert_provider_account( + store, + UpsertProviderAccountInput { + provider, + provider_user_id: observed.provider_user_id.as_deref(), + email: observed.email.as_deref(), + label: None, + // Plan evidence has its own history and must not mutate billing/account facts. + plan_name: None, + identity_source: Some(IdentitySource::LocalAuth), + verified_at: Some(observed.observed_at), + }, + )?; + canonical_ids.insert(detected_id, account.provider_account_id); + } + + remap_account_evidence_account_ids(evidence, &canonical_ids); + Ok(()) +} + +fn canonicalize_known_account_evidence( + store: &Store, + provider: &str, + evidence: &mut AccountEvidenceScan, +) -> Result> { + let mut canonical_ids = HashMap::new(); + for observed in &evidence.accounts { + let Some(detected_id) = provider_account_id_from_identity( + provider, + observed.provider_user_id.as_deref(), + observed.email.as_deref(), + ) else { + continue; + }; + if let Some(account) = find_existing_provider_account( + store, + provider, + observed.provider_user_id.as_deref(), + observed.email.as_deref(), + )? { + canonical_ids.insert(detected_id, account.provider_account_id); + } + } + remap_account_evidence_account_ids(evidence, &canonical_ids); + Ok(canonical_ids) +} + fn apply_source_account_resolution( store: &Store, source: &SourceLocation, @@ -9504,6 +9838,8 @@ fn sanitize_account_for_sync(mut account: ProviderAccount) -> ProviderAccount { if !matches!(account.identity_source, IdentitySource::UserConfigured) { account.account_label = None; } + account.provider_user_id = None; + account.email = None; account.plan_name = None; account } @@ -9892,6 +10228,293 @@ mod tests { } } + struct AccountEvidenceTrackingAdapter { + source: SourceLocation, + candidate: ScanCandidateFile, + event: UsageEvent, + collect_calls: Arc>, + } + + impl ProviderAdapter for AccountEvidenceTrackingAdapter { + fn id(&self) -> &'static str { + "test-account-evidence" + } + + fn version(&self) -> &'static str { + "0" + } + + fn provider(&self) -> &'static str { + "codex" + } + + fn discover(&self) -> Vec { + vec![self.source.clone()] + } + + fn scan_candidates(&self, _source: &SourceLocation) -> Result> { + Ok(vec![self.candidate.clone()]) + } + + fn collect_account_evidence( + &self, + _source: &SourceLocation, + _checkpoints: &[statsai_core::AccountEvidenceCheckpointV1], + ) -> Result { + *self.collect_calls.lock().expect("collect calls") += 1; + Ok(AccountEvidenceScan::default()) + } + + fn scan( + &self, + _source: &SourceLocation, + _options: &ScanOptions, + ) -> Result { + Ok(statsai_adapters::AdapterScan { + events: vec![self.event.clone()], + ..statsai_adapters::AdapterScan::default() + }) + } + } + + fn seed_test_account_evidence( + store: &Store, + source: &SourceLocation, + observed_at: DateTime, + ) { + let account_id = ProviderAccountId("account-source-cleanup".to_string()); + store + .upsert_account_identity_observations(&[statsai_core::AccountIdentityObservationV1 { + schema_version: statsai_core::ACCOUNT_IDENTITY_OBSERVATION_SCHEMA_VERSION + .to_string(), + observation_id: "identity-source-cleanup".to_string(), + provider: source.provider.clone(), + source_id: source.source_id.clone(), + provider_account_id: Some(account_id.clone()), + provider_user_id_hash: Some("a".repeat(64)), + email_hash: None, + conversation_id_hash: Some("b".repeat(64)), + turn_id_hash: None, + observed_at, + evidence_kind: statsai_core::AccountEvidenceKind::TelemetryIdentity, + confidence: Confidence::High, + auth_mode: Some("chatgpt".to_string()), + application_version: None, + parser_version: "test.v1".to_string(), + artifact_kind: "test".to_string(), + artifact_path_hash: "c".repeat(64), + record_fingerprint: "d".repeat(64), + }]) + .expect("identity evidence"); + store + .upsert_account_plan_observations(&[statsai_core::AccountPlanObservationV1 { + schema_version: statsai_core::ACCOUNT_PLAN_OBSERVATION_SCHEMA_VERSION.to_string(), + observation_id: "plan-source-cleanup".to_string(), + provider: source.provider.clone(), + source_id: source.source_id.clone(), + provider_account_id: Some(account_id.clone()), + raw_plan_name: "pro".to_string(), + plan_name: "Pro".to_string(), + observed_at, + active_from: None, + active_until: None, + is_current_snapshot: false, + evidence_kind: statsai_core::AccountEvidenceKind::QuotaStatus, + confidence: Confidence::High, + parser_version: "test.v1".to_string(), + artifact_path_hash: "c".repeat(64), + record_fingerprint: "e".repeat(64), + }]) + .expect("plan evidence"); + store + .upsert_conversation_account_bindings(&[statsai_core::ConversationAccountBindingV1 { + schema_version: statsai_core::CONVERSATION_ACCOUNT_BINDING_SCHEMA_VERSION + .to_string(), + binding_id: "binding-source-cleanup".to_string(), + provider: source.provider.clone(), + source_id: source.source_id.clone(), + provider_account_id: account_id, + conversation_id_hash: "b".repeat(64), + turn_id_hash: None, + observed_at, + evidence_kind: statsai_core::AccountEvidenceKind::ResetHistory, + confidence: Confidence::High, + }]) + .expect("conversation evidence"); + } + + #[test] + fn canonicalization_skips_accounts_without_surviving_evidence() { + let store = Store::in_memory().expect("store"); + let mut evidence = AccountEvidenceScan::default(); + evidence + .accounts + .push(statsai_adapters::ObservedProviderAccount { + provider_user_id: Some("unchanged-provider-user".to_string()), + email: Some("unchanged@example.test".to_string()), + plan_name: None, + observed_at: Utc + .with_ymd_and_hms(2026, 8, 23, 10, 12, 43) + .single() + .expect("date"), + }); + + retain_accounts_referenced_by_account_evidence("codex", &HashMap::new(), &mut evidence); + canonicalize_account_evidence(&store, "codex", &mut evidence) + .expect("canonicalize account evidence"); + + assert!(evidence.accounts.is_empty()); + assert!(store.list_accounts().expect("accounts").is_empty()); + } + + #[test] + fn known_account_aliases_are_applied_before_evidence_deduplication() { + let store = Store::in_memory().expect("store"); + let observed_at = Utc + .with_ymd_and_hms(2026, 8, 23, 10, 12, 43) + .single() + .expect("date"); + upsert_provider_account( + &store, + UpsertProviderAccountInput { + provider: "codex", + provider_user_id: None, + email: Some("owner@example.test"), + label: None, + plan_name: None, + identity_source: Some(IdentitySource::LocalAuth), + verified_at: Some(observed_at), + }, + ) + .expect("email-only account"); + let source_id = SourceId("alias-dedup-source".to_string()); + let raw_account_id = provider_account_id_from_identity( + "codex", + Some("provider-user-1"), + Some("owner@example.test"), + ) + .expect("detected account id"); + let raw_plan = statsai_core::AccountPlanObservationV1 { + schema_version: statsai_core::ACCOUNT_PLAN_OBSERVATION_SCHEMA_VERSION.to_string(), + observation_id: account_plan_observation_id( + &source_id, + Some(&raw_account_id), + "plus", + observed_at, + statsai_core::AccountEvidenceKind::AuthSnapshot, + ), + provider: "codex".to_string(), + source_id: source_id.clone(), + provider_account_id: Some(raw_account_id.clone()), + raw_plan_name: "plus".to_string(), + plan_name: "Plus".to_string(), + observed_at, + active_from: None, + active_until: None, + is_current_snapshot: true, + evidence_kind: statsai_core::AccountEvidenceKind::AuthSnapshot, + confidence: Confidence::High, + parser_version: "test.v1".to_string(), + artifact_path_hash: "a".repeat(64), + record_fingerprint: "b".repeat(64), + }; + let raw_binding = statsai_core::ConversationAccountBindingV1 { + schema_version: statsai_core::CONVERSATION_ACCOUNT_BINDING_SCHEMA_VERSION.to_string(), + binding_id: conversation_account_binding_id( + &source_id, + &"c".repeat(64), + None, + &raw_account_id, + ), + provider: "codex".to_string(), + source_id: source_id.clone(), + provider_account_id: raw_account_id, + conversation_id_hash: "c".repeat(64), + turn_id_hash: None, + observed_at, + evidence_kind: statsai_core::AccountEvidenceKind::ResetHistory, + confidence: Confidence::High, + }; + let raw_scan = AccountEvidenceScan { + accounts: vec![statsai_adapters::ObservedProviderAccount { + provider_user_id: Some("provider-user-1".to_string()), + email: Some("owner@example.test".to_string()), + plan_name: None, + observed_at, + }], + plan_observations: vec![raw_plan], + conversation_bindings: vec![raw_binding], + ..AccountEvidenceScan::default() + }; + + let mut first_scan = raw_scan.clone(); + canonicalize_account_evidence(&store, "codex", &mut first_scan) + .expect("canonicalize first scan"); + store + .upsert_account_plan_observations(&first_scan.plan_observations) + .expect("store canonical plan"); + store + .upsert_conversation_account_bindings(&first_scan.conversation_bindings) + .expect("store canonical binding"); + + let mut repeated_scan = raw_scan; + let known_account_aliases = + canonicalize_known_account_evidence(&store, "codex", &mut repeated_scan) + .expect("canonicalize known aliases"); + store + .retain_unseen_account_evidence( + &source_id, + &mut repeated_scan.identity_observations, + &mut repeated_scan.plan_observations, + &mut repeated_scan.conversation_bindings, + ) + .expect("filter canonical evidence"); + assert!(repeated_scan.plan_observations.is_empty()); + assert!(repeated_scan.conversation_bindings.is_empty()); + + repeated_scan + .identity_observations + .push(statsai_core::AccountIdentityObservationV1 { + schema_version: statsai_core::ACCOUNT_IDENTITY_OBSERVATION_SCHEMA_VERSION + .to_string(), + observation_id: "identity-provider-user-enrichment".to_string(), + provider: "codex".to_string(), + source_id, + provider_account_id: known_account_aliases.values().next().cloned(), + provider_user_id_hash: Some("provider-user-hash".to_string()), + email_hash: Some("email-hash".to_string()), + conversation_id_hash: None, + turn_id_hash: None, + observed_at, + evidence_kind: statsai_core::AccountEvidenceKind::AuthSnapshot, + confidence: Confidence::High, + auth_mode: Some("chatgpt".to_string()), + application_version: None, + parser_version: "test.v1".to_string(), + artifact_kind: "auth_json".to_string(), + artifact_path_hash: "d".repeat(64), + record_fingerprint: "e".repeat(64), + }); + retain_accounts_referenced_by_account_evidence( + "codex", + &known_account_aliases, + &mut repeated_scan, + ); + assert_eq!( + repeated_scan.accounts.len(), + 1, + "the account carrying a newly learned provider ID must survive canonical alias remapping" + ); + canonicalize_account_evidence(&store, "codex", &mut repeated_scan) + .expect("enrich canonical account"); + let accounts = store.list_accounts().expect("accounts"); + assert_eq!(accounts.len(), 1); + assert_eq!( + accounts[0].provider_user_id.as_deref(), + Some("provider-user-1") + ); + } + #[derive(Clone)] struct AttributionBlockedTestAdapter { provider: &'static str, @@ -11376,6 +11999,78 @@ mod tests { assert_eq!(assignments[1].started_at, second_start); } + #[test] + fn manual_source_reassignment_rebuilds_quota_plan_evidence() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/codex-quota-plan-reassignment"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let mut quota = test_unattributed_quota_record(&source.source_id.0); + quota.observation.status.plan_type = Some("pro".to_string()); + store + .upsert_quota_observations(&[quota]) + .expect("quota observation"); + let first_start = Utc + .with_ymd_and_hms(2026, 8, 1, 0, 0, 0) + .single() + .expect("first start"); + let second_start = Utc + .with_ymd_and_hms(2026, 8, 15, 0, 0, 0) + .single() + .expect("second start"); + + let first = connect_source_to_account( + &store, + ConnectSourceToAccountInput { + source_id: &source.source_id, + provider_account_id_value: None, + provider_user_id: None, + email: Some("first-quota@example.com"), + label: None, + started_at: first_start, + ended_at: None, + }, + ) + .expect("first connection"); + store + .rebuild_quota_plan_observations_for_source(&source.source_id) + .expect("seed quota plan evidence"); + assert_eq!( + store.account_plan_observations().expect("initial plan")[0].provider_account_id, + Some(first.provider_account_id) + ); + + let second = connect_source_to_account( + &store, + ConnectSourceToAccountInput { + source_id: &source.source_id, + provider_account_id_value: None, + provider_user_id: None, + email: Some("second-quota@example.com"), + label: None, + started_at: second_start, + ended_at: None, + }, + ) + .expect("second connection"); + + let observations = store.account_plan_observations().expect("reassigned plan"); + assert_eq!(observations.len(), 1); + assert_eq!( + observations[0].provider_account_id, + Some(second.provider_account_id) + ); + assert_eq!( + observations[0].evidence_kind, + statsai_core::AccountEvidenceKind::QuotaStatus + ); + } + #[test] fn connect_source_to_account_preserves_tail_when_replacing_finite_window() { let store = Store::in_memory().expect("store"); @@ -11641,7 +12336,7 @@ mod tests { accounts[0].provider_user_id.as_deref(), Some("chatgpt-account-123") ); - assert_eq!(accounts[0].plan_name.as_deref(), Some("Plus")); + assert_eq!(accounts[0].plan_name, None); assert_eq!(accounts[0].verified_at, Some(verified_at)); let assignments = store @@ -11655,18 +12350,10 @@ mod tests { assert_eq!(assignments[0].record_source, IdentitySource::LocalAuth); assert_eq!(assignments[0].verified_at, Some(verified_at)); - let subscriptions = store.list_subscriptions().expect("subscriptions"); - assert_eq!(subscriptions.len(), 1); - assert_eq!( - subscriptions[0].provider_account_id, - existing.provider_account_id - ); - assert_eq!(subscriptions[0].record_source, IdentitySource::LocalAuth); - assert_eq!( - subscriptions[0].current_period_ends_at, - Some(current_period_ends_at) - ); - assert_eq!(subscriptions[0].ended_at, None); + assert!(store + .list_subscriptions() + .expect("subscriptions") + .is_empty()); } #[test] @@ -11947,7 +12634,7 @@ mod tests { assert!(accounts.iter().any(|account| { account.provider_account_id == expected_account_id && account.email.as_deref() == Some("verified@example.com") - && account.plan_name.as_deref() == Some("Plus") + && account.plan_name.is_none() })); let assignments = store @@ -11959,15 +12646,10 @@ mod tests { assert_eq!(assignments[0].provider_account_id, expected_account_id); assert_eq!(assignments[0].record_source, IdentitySource::LocalAuth); - let subscriptions = store.list_subscriptions().expect("subscriptions"); - assert_eq!(subscriptions.len(), 1); - assert_eq!(subscriptions[0].provider_account_id, expected_account_id); - assert_eq!(subscriptions[0].record_source, IdentitySource::LocalAuth); - assert_eq!(subscriptions[0].ended_at, None); - assert_eq!( - subscriptions[0].current_period_ends_at, - Some(current_period_ends_at) - ); + assert!(store + .list_subscriptions() + .expect("subscriptions") + .is_empty()); let stored_source = store .source(&source.source_id) .expect("source row") @@ -12747,6 +13429,62 @@ mod tests { .is_empty()); } + #[test] + fn manual_only_source_does_not_collect_or_apply_account_evidence() { + let store = Store::in_memory().expect("store"); + let source_root = "/tmp/codex-manual-only-evidence"; + let file_path = "/tmp/codex-manual-only-evidence/session.jsonl"; + let mut source = SourceLocation::local_adapter( + "codex", + "test-account-evidence", + "0", + Path::new(source_root), + LocationOrigin::Configured, + ); + source.verification_mode = SourceVerificationMode::ManualOnly; + store.upsert_source(&source).expect("source"); + let observed_at = Utc + .with_ymd_and_hms(2026, 8, 23, 10, 0, 0) + .single() + .expect("observed at"); + seed_test_account_evidence(&store, &source, observed_at); + + let mut event = test_scan_event(&source, file_path, observed_at, "manual-event", 100); + event.session.local_session_id_hash = Some("b".repeat(64)); + let collect_calls = Arc::new(Mutex::new(0)); + let adapter = AccountEvidenceTrackingAdapter { + source: source.clone(), + candidate: test_scan_candidate(file_path, "manual-evidence-v1"), + event, + collect_calls: Arc::clone(&collect_calls), + }; + + scan_with_adapters( + ScanCommand { + provider: None, + include_tasks: false, + preview: false, + no_cache: false, + replace: false, + verbose: false, + explain: false, + }, + &store, + "device-test", + vec![Box::new(adapter)], + ) + .expect("scan"); + + assert_eq!(*collect_calls.lock().expect("collect calls"), 0); + let stored_event = store + .events() + .expect("events") + .into_iter() + .find(|item| item.source.source_record_id.as_deref() == Some("manual-event")) + .expect("manual event"); + assert_eq!(stored_event.provider_account_id, None); + } + #[test] fn disabled_source_mode_closes_verified_linkages() { let store = Store::in_memory().expect("store"); @@ -12818,6 +13556,7 @@ mod tests { notes: None, }) .expect("subscription"); + seed_test_account_evidence(&store, &source_location, started_at); source( SourceCommand { @@ -12847,6 +13586,18 @@ mod tests { assert!(assignments[0].ended_at.is_some()); assert_eq!(subscriptions.len(), 1); assert!(subscriptions[0].ended_at.is_some()); + assert!(store + .account_identity_observations(Some(&source.source_id)) + .expect("identity evidence") + .is_empty()); + assert!(store + .account_plan_observations() + .expect("plan evidence") + .is_empty()); + assert!(store + .conversation_account_bindings(Some(&source.source_id)) + .expect("conversation evidence") + .is_empty()); } #[test] @@ -12957,6 +13708,7 @@ mod tests { .with_ymd_and_hms(2026, 6, 1, 10, 0, 0) .single() .expect("now"); + seed_test_account_evidence(&store, &committed_source, now); let mut summary = test_summary("codex", &committed_source, now, 100, None); summary.project = Some(ProjectInfo { project_id: "project-committed-only".to_string(), @@ -13002,6 +13754,18 @@ mod tests { .list_code_change_metrics(false) .expect("metrics after") .is_empty()); + assert!(store + .account_identity_observations(Some(&committed_source.source_id)) + .expect("identity evidence after") + .is_empty()); + assert!(store + .account_plan_observations() + .expect("plan evidence after") + .is_empty()); + assert!(store + .conversation_account_bindings(Some(&committed_source.source_id)) + .expect("conversation evidence after") + .is_empty()); } #[test] @@ -14394,6 +15158,8 @@ mod tests { accounts: vec![], source_account_assignments: vec![], subscriptions: vec![], + account_plan_observations: vec![], + account_evidence_summaries: vec![], events: vec![], summaries, task_buckets: vec![], @@ -14443,6 +15209,8 @@ mod tests { accounts: Vec::new(), source_account_assignments: Vec::new(), subscriptions: Vec::new(), + account_plan_observations: Vec::new(), + account_evidence_summaries: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: Vec::new(), @@ -14495,6 +15263,8 @@ mod tests { accounts: Vec::new(), source_account_assignments: Vec::new(), subscriptions: Vec::new(), + account_plan_observations: Vec::new(), + account_evidence_summaries: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: Vec::new(), @@ -14615,6 +15385,8 @@ mod tests { accounts, source_account_assignments: assignments, subscriptions, + account_plan_observations: vec![], + account_evidence_summaries: vec![], events: vec![], summaries, task_buckets: vec![], @@ -14683,6 +15455,8 @@ mod tests { accounts: vec![], source_account_assignments: vec![], subscriptions: vec![], + account_plan_observations: vec![], + account_evidence_summaries: vec![], events: vec![], summaries, task_buckets: vec![], @@ -14776,6 +15550,8 @@ mod tests { accounts: vec![], source_account_assignments: vec![], subscriptions: vec![], + account_plan_observations: vec![], + account_evidence_summaries: vec![], events: vec![], summaries, task_buckets: vec![], @@ -15133,6 +15909,8 @@ mod tests { accounts: accounts.clone(), source_account_assignments: vec![], subscriptions: vec![], + account_plan_observations: vec![], + account_evidence_summaries: vec![], events: vec![], summaries: vec![], task_buckets: vec![], @@ -15479,6 +16257,8 @@ mod tests { accounts: vec![], source_account_assignments: vec![], subscriptions: vec![], + account_plan_observations: vec![], + account_evidence_summaries: vec![], events: vec![], summaries: vec![], task_buckets: test_task_only_sync_batch(now, 1, 0).task_buckets, @@ -15564,6 +16344,8 @@ mod tests { accounts: vec![], source_account_assignments: vec![], subscriptions: vec![], + account_plan_observations: vec![], + account_evidence_summaries: vec![], events: vec![], summaries, task_buckets: vec![], @@ -15617,6 +16399,53 @@ mod tests { ); } + #[test] + fn v4_account_evidence_d1_estimate_includes_alias_lookup() { + let now = Utc + .with_ymd_and_hms(2026, 8, 23, 10, 12, 43) + .single() + .expect("date"); + let mut batch = test_task_only_sync_batch(now, 0, 0); + let baseline = estimate_http_rollup_d1_queries(&batch); + let account_id = ProviderAccountId("account-plan-estimate".to_string()); + batch.account_plan_observations = vec![statsai_core::AccountPlanProjectionV1 { + schema_version: statsai_core::ACCOUNT_PLAN_PROJECTION_SCHEMA_VERSION.to_string(), + projection_id: "projection-plan-estimate".to_string(), + semantic_fingerprint: "a".repeat(64), + device_id: batch.device_id.clone(), + provider: "codex".to_string(), + provider_account_id: account_id.clone(), + raw_plan_name: "plus".to_string(), + plan_name: "Plus".to_string(), + observed_at: now, + active_from: None, + active_until: None, + is_current_snapshot: true, + evidence_kind: statsai_core::AccountEvidenceKind::AuthSnapshot, + confidence: Confidence::High, + }]; + batch.account_evidence_summaries = vec![statsai_core::AccountEvidenceSummaryV1 { + schema_version: statsai_core::ACCOUNT_EVIDENCE_SUMMARY_SCHEMA_VERSION.to_string(), + summary_id: "evidence-summary-estimate".to_string(), + device_id: batch.device_id.clone(), + provider: "codex".to_string(), + provider_account_id: account_id, + first_strong_observed_at: Some(now), + last_strong_observed_at: Some(now), + strong_observation_count: 1, + directly_bound_conversations: 0, + uncovered_gap_count: 0, + conflict_count: 0, + evidence_kinds: vec![statsai_core::AccountEvidenceKind::AuthSnapshot], + }]; + + assert_eq!( + estimate_http_rollup_d1_queries(&batch), + baseline + 3, + "two metadata writes and one evidence-alias lookup must be budgeted" + ); + } + #[test] fn code_change_metrics_use_the_backends_batched_collection_limit() { let now = Utc @@ -15666,6 +16495,8 @@ mod tests { accounts: vec![], source_account_assignments: vec![], subscriptions: vec![], + account_plan_observations: vec![], + account_evidence_summaries: vec![], events: vec![], summaries: vec![summary], task_buckets: vec![], @@ -15814,6 +16645,8 @@ mod tests { accounts: Vec::new(), source_account_assignments: Vec::new(), subscriptions: Vec::new(), + account_plan_observations: Vec::new(), + account_evidence_summaries: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets, @@ -15927,6 +16760,8 @@ mod tests { accounts: Vec::new(), source_account_assignments: Vec::new(), subscriptions: Vec::new(), + account_plan_observations: Vec::new(), + account_evidence_summaries: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: vec![TaskBucketSnapshot { @@ -16127,6 +16962,8 @@ mod tests { accounts: Vec::new(), source_account_assignments: Vec::new(), subscriptions: Vec::new(), + account_plan_observations: Vec::new(), + account_evidence_summaries: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets, @@ -16474,9 +17311,7 @@ mod tests { &local_verify ) .as_deref(), - Some( - "sources 0!=1, accounts 0!=1, source_account_assignments 0!=1, subscriptions 0!=1" - ) + Some("sources 0!=1, accounts 0!=1, source_account_assignments 0!=1") ); store @@ -16488,7 +17323,7 @@ mod tests { assert_eq!(batch.sources.len(), 1); assert_eq!(batch.accounts.len(), 1); assert_eq!(batch.source_account_assignments.len(), 1); - assert_eq!(batch.subscriptions.len(), 1); + assert!(batch.subscriptions.is_empty()); assert_eq!(batch.summaries.len(), 1); assert!(is_daily_rollup_summary(&batch.summaries[0])); } @@ -18920,6 +19755,8 @@ mod tests { accounts: Vec::new(), source_account_assignments: Vec::new(), subscriptions: Vec::new(), + account_plan_observations: Vec::new(), + account_evidence_summaries: Vec::new(), events: Vec::new(), summaries: Vec::new(), task_buckets: Vec::new(), @@ -19588,6 +20425,154 @@ mod tests { ); } + #[test] + fn quota_contributions_reach_the_batch_and_its_authoritative_ids() { + let store = Store::in_memory().expect("store"); + let source = SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/codex-quota-v4-sync"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let observed_at = Utc + .with_ymd_and_hms(2026, 8, 20, 12, 0, 0) + .single() + .expect("observed at"); + let account_id = ProviderAccountId("account-quota-sync".to_string()); + store + .upsert_source_account_assignment(&SourceAccountAssignment { + schema_version: SOURCE_ACCOUNT_ASSIGNMENT_SCHEMA_VERSION.to_string(), + assignment_id: SourceAccountAssignmentId("assignment-quota-sync".to_string()), + source_id: source.source_id.clone(), + provider: "codex".to_string(), + provider_account_id: account_id, + started_at: observed_at - Duration::days(1), + ended_at: None, + record_source: IdentitySource::UserConfigured, + verified_at: Some(observed_at), + created_at: observed_at, + updated_at: observed_at, + }) + .expect("assignment"); + let reset_at = observed_at + Duration::days(7); + let quota_record: QuotaObservationRecordV1 = serde_json::from_value(json!({ + "observation": { + "schema_version": "quota_observation.v1", + "observation_id": "quota-observation-sync", + "semantic_fingerprint": "quota-semantic-sync", + "provider": "codex", + "source_id": source.source_id, + "provider_account_id": null, + "observed_at": observed_at, + "source_file_path_hash": "file-hash", + "source_record_id": "record-id", + "source_line_number": 1, + "payload_hash": "payload-hash", + "usage_sample": null, + "usage_event_id": null, + "usage_link_kind": "none", + "status": { + "plan_type": "pro", + "individual_limit": { + "account_email": "private@example.com", + "nested": {"token": "provider-secret"} + }, + "spend_control_state": null, + "reached_type": null, + "credits": { + "has_credits": false, + "unlimited": false, + "balance": null, + "balance_raw": null + } + } + }, + "windows": [{ + "schema_version": "quota_window_observation.v1", + "window_observation_id": "quota-window-sync", + "observation_id": "quota-observation-sync", + "provider_slot": "primary", + "limit_id": "subscription", + "window_minutes": 10080, + "used_percent": 25.0, + "resets_at": reset_at, + "resets_at_epoch_seconds": reset_at.timestamp() + }], + "raw_rate_limits": {"primary": {"used_percent": 25.0}} + })) + .expect("quota record"); + store + .upsert_quota_observations(&[quota_record]) + .expect("quota observation"); + + let command = SyncCommand { + dry_run: true, + endpoint: Some("https://api.example.com/api/sync/batches".to_string()), + ..test_sync_command("http") + }; + let target = sync_target(&command).expect("target"); + let (batch, _) = + build_sync_batch(&command, &store, "device-quota", &target).expect("v4 quota batch"); + + assert_eq!( + batch.schema_version, + statsai_core::SYNC_BATCH_V5_SCHEMA_VERSION + ); + assert_eq!(batch.quota_cycle_contributions.len(), 1); + // The uploaded contribution carries no provider status at all, so the + // plan, credits, and `individual_limit` blob a window observation holds + // cannot reach the backend even by accident. + let uploaded = serde_json::to_value(&batch.quota_cycle_contributions[0]) + .expect("serialize contribution"); + assert_eq!(uploaded.get("latest_status"), None); + let contribution_id = batch.quota_cycle_contributions[0].contribution_id.clone(); + assert_eq!( + batch + .authoritative_snapshot + .as_ref() + .expect("authoritative snapshot") + .quota_cycle_contribution_ids, + vec![contribution_id.clone()] + ); + let chunks = split_http_rollup_sync_batches(&batch); + assert!(chunks.iter().any(|chunk| { + chunk + .quota_cycle_contributions + .iter() + .any(|contribution| contribution.contribution_id == contribution_id) + })); + assert!(chunks.iter().any(|chunk| { + chunk + .authoritative_snapshot + .as_ref() + .is_some_and(|snapshot| { + snapshot + .quota_cycle_contribution_ids + .contains(&contribution_id) + }) + })); + + record_rollup_sync_success(&store, "http", &target, &batch) + .expect("record initial quota sync"); + let (unchanged, _) = build_sync_batch(&command, &store, "device-quota", &target) + .expect("unchanged quota batch"); + assert!(unchanged.quota_cycle_contributions.is_empty()); + assert!(unchanged.authoritative_snapshot.is_none()); + + store + .delete_quota_observations_for_sources(std::slice::from_ref(&source.source_id)) + .expect("delete quota evidence"); + let (retirement, _) = build_sync_batch(&command, &store, "device-quota", &target) + .expect("quota retirement batch"); + assert!(retirement.quota_cycle_contributions.is_empty()); + assert!(retirement + .authoritative_snapshot + .as_ref() + .is_some_and(|snapshot| snapshot.quota_cycle_contribution_ids.is_empty())); + } + #[test] fn sanitize_account_for_sync_preserves_user_configured_label() { let account = ProviderAccount { @@ -19595,10 +20580,10 @@ mod tests { provider_account_id: provider_account_id("codex", "personal"), provider: "codex".to_string(), identity_source: IdentitySource::UserConfigured, - provider_user_id: None, - provider_user_id_hash: None, - email: None, - email_hash: None, + provider_user_id: Some("provider-user-secret".to_string()), + provider_user_id_hash: Some("a".repeat(64)), + email: Some("private@example.com".to_string()), + email_hash: Some("b".repeat(64)), org_id_hash: None, account_label: Some("personal".to_string()), plan_name: Some("Pro".to_string()), @@ -19610,7 +20595,21 @@ mod tests { let sanitized = sanitize_account_for_sync(account); assert_eq!(sanitized.account_label.as_deref(), Some("personal")); + assert_eq!(sanitized.provider_user_id, None); + assert_eq!(sanitized.email, None); + assert_eq!( + sanitized.provider_user_id_hash.as_deref(), + Some("a".repeat(64).as_str()) + ); + assert_eq!( + sanitized.email_hash.as_deref(), + Some("b".repeat(64).as_str()) + ); assert_eq!(sanitized.plan_name, None); + + let payload = serde_json::to_string(&sanitized).expect("serialize sanitized account"); + assert!(!payload.contains("provider-user-secret")); + assert!(!payload.contains("private@example.com")); } #[test] @@ -19766,6 +20765,76 @@ mod tests { }); store.insert_event(&event).expect("event"); store.upsert_summary(&summary).expect("summary"); + let identity = statsai_core::AccountIdentityObservationV1 { + schema_version: statsai_core::ACCOUNT_IDENTITY_OBSERVATION_SCHEMA_VERSION.to_string(), + observation_id: "identity-merge-alias".to_string(), + provider: "codex".to_string(), + source_id: source.source_id.clone(), + provider_account_id: Some(alias.provider_account_id.clone()), + provider_user_id_hash: Some("a".repeat(64)), + email_hash: None, + conversation_id_hash: Some("b".repeat(64)), + turn_id_hash: None, + observed_at: now, + evidence_kind: statsai_core::AccountEvidenceKind::TelemetryIdentity, + confidence: Confidence::High, + auth_mode: Some("chatgpt".to_string()), + application_version: None, + parser_version: "test.v1".to_string(), + artifact_kind: "test".to_string(), + artifact_path_hash: "c".repeat(64), + record_fingerprint: "d".repeat(64), + }; + let plan = statsai_core::AccountPlanObservationV1 { + schema_version: statsai_core::ACCOUNT_PLAN_OBSERVATION_SCHEMA_VERSION.to_string(), + observation_id: account_plan_observation_id( + &source.source_id, + Some(&alias.provider_account_id), + "plus", + now, + statsai_core::AccountEvidenceKind::AuthSnapshot, + ), + provider: "codex".to_string(), + source_id: source.source_id.clone(), + provider_account_id: Some(alias.provider_account_id.clone()), + raw_plan_name: "plus".to_string(), + plan_name: "Plus".to_string(), + observed_at: now, + active_from: None, + active_until: None, + is_current_snapshot: true, + evidence_kind: statsai_core::AccountEvidenceKind::AuthSnapshot, + confidence: Confidence::High, + parser_version: "test.v1".to_string(), + artifact_path_hash: "c".repeat(64), + record_fingerprint: "e".repeat(64), + }; + let binding = statsai_core::ConversationAccountBindingV1 { + schema_version: statsai_core::CONVERSATION_ACCOUNT_BINDING_SCHEMA_VERSION.to_string(), + binding_id: conversation_account_binding_id( + &source.source_id, + &"b".repeat(64), + None, + &alias.provider_account_id, + ), + provider: "codex".to_string(), + source_id: source.source_id.clone(), + provider_account_id: alias.provider_account_id.clone(), + conversation_id_hash: "b".repeat(64), + turn_id_hash: None, + observed_at: now, + evidence_kind: statsai_core::AccountEvidenceKind::ResetHistory, + confidence: Confidence::High, + }; + store + .upsert_account_identity_observations(std::slice::from_ref(&identity)) + .expect("identity evidence"); + store + .upsert_account_plan_observations(std::slice::from_ref(&plan)) + .expect("plan evidence"); + store + .upsert_conversation_account_bindings(std::slice::from_ref(&binding)) + .expect("conversation binding"); let target = "https://api.example.com/api/sync/batches"; store @@ -19802,6 +20871,9 @@ mod tests { assert_eq!(report.moved_subscriptions, 0); assert_eq!(report.moved_events, 1); assert_eq!(report.moved_summaries, 1); + assert_eq!(report.moved_identity_observations, 1); + assert_eq!(report.moved_plan_observations, 1); + assert_eq!(report.moved_conversation_bindings, 1); assert!(report.deleted_source_account); assert!(report.reset_local_sync_tracking); assert_eq!(report.remaining_references.total(), 0); @@ -19835,6 +20907,29 @@ mod tests { summaries[0].provider_account_id, Some(canonical.provider_account_id.clone()) ); + assert!(store + .account_identity_observations(None) + .expect("identity evidence") + .iter() + .all(|observation| { + observation.provider_account_id.as_ref() == Some(&canonical.provider_account_id) + })); + let plans = store.account_plan_observations().expect("plan evidence"); + assert_eq!(plans.len(), 1); + assert_eq!( + plans[0].provider_account_id.as_ref(), + Some(&canonical.provider_account_id) + ); + assert_ne!(plans[0].observation_id, plan.observation_id); + let bindings = store + .conversation_account_bindings(None) + .expect("conversation bindings"); + assert_eq!(bindings.len(), 1); + assert_eq!( + bindings[0].provider_account_id, + canonical.provider_account_id + ); + assert_ne!(bindings[0].binding_id, binding.binding_id); assert!(store.list_sync_states().expect("sync states").is_empty()); let sync_accounts: Vec<_> = store diff --git a/docs/sync-contract.md b/docs/sync-contract.md index d50c147..d57c74f 100644 --- a/docs/sync-contract.md +++ b/docs/sync-contract.md @@ -10,6 +10,12 @@ contract. Versions 1 and 2 never treat those new fields as authoritative. `sync_batch.v4` adds provider-neutral quota-cycle contributions and makes their acknowledgement counts and authoritative snapshot IDs part of the versioned contract. Versions 1–3 never treat those new fields as authoritative. +`sync_batch.v5` adds privacy-safe account-plan observations and aggregate +evidence-quality summaries, and makes their acknowledgement counts and +authoritative snapshot IDs part of the versioned contract. Versions 1–4 never +treat those collections or their snapshot IDs as authoritative. Because a v4 +acknowledgement carries no counter for them, a collector must not place them in +a v4 batch even though a v4 backend would ignore the unknown keys. The collector owns local scanning, normalization, idempotent local storage, and privacy scrubbing. The backend owns authentication, validation, deduplication, rollups, and user-facing queries. The production path sends sanitized batches to @@ -89,6 +95,11 @@ The current production sync path strips record-level local evidence before sendi - `Subscription.notes` - code source text, diffs, file paths, tool arguments, and commit messages +Account-plan sync sends only the canonical account reference, provider, raw and +display plan names, observation time, explicit provider bounds, evidence kind, +confidence, and aggregate coverage/conflict counts. Email, provider user ID, +conversation and turn IDs, artifact paths, tokens, and raw provenance remain local. + Code-change counting uses an explicit source-file extension allowlist. Documentation, configuration, manifests, lockfiles, generated output, and unknown text formats are excluded. Mutation records that claim a whole-file @@ -105,9 +116,10 @@ are removed while the retained conversation archive remains available locally. Reconciliation is skipped when the archive root itself is unreachable, because an unmounted volume or a renamed home directory produces the same empty file list as an emptied archive and must not be read as a deletion. Removing a source -with `--delete-data` drops that source's reconstructed edits and coverage and -rebuilds the metrics immediately, so the authoritative snapshot retires the -hosted rows instead of continuing to republish them. +with `--delete-data` drops that source's reconstructed edits, coverage, identity +observations, plan observations, and conversation bindings, then rebuilds the +derived metrics immediately. The next authoritative snapshot therefore retires +hosted rows instead of continuing to republish deleted source evidence. An edit is placed in a repository only when its path stays inside that repository. A tool call naming an absolute path elsewhere, or escaping the @@ -250,16 +262,17 @@ device's cycles would tell the sender they had been stored when they had not. - require an authenticated device access token - accept `Authorization: Bearer ` from stored auth, `--auth-token`, or `STATSAI_SYNC_TOKEN` -- validate the request body against `sync_batch.v1`, `sync_batch.v2`, `sync_batch.v3`, and `sync_batch.v4` +- validate the request body against `sync_batch.v1` through `sync_batch.v5` - reject unsupported `schema_version` values -- deduplicate sources, accounts, source-account assignments, subscriptions, and summaries by their IDs when server-side deduplication is needed +- deduplicate sources, accounts, source-account assignments, subscriptions, summaries, and equivalent account-plan evidence when server-side deduplication is needed - treat collector IDs as stable client-provided IDs, not database primary keys exposed to users - compute daily, monthly, and dashboard rollups server-side from accepted summaries - atomically replace accepted task bucket snapshots per `(user, device, project_bucket)` - treat the ordered `authoritative_snapshot` fragments sharing one `snapshot_id` as the complete set of metadata and summary IDs owned by the authenticated device; v3 fragments also carry code-change metric IDs; v4 fragments also - carry quota-cycle contribution IDs; each fragment carries + carry quota-cycle contribution IDs; v5 fragments also carry account-plan + observation and evidence-summary IDs; each fragment carries zero-based `part_index` and a common `part_count`, with at most 200 IDs across its ID arrays - stage snapshot ownership without pruning until the final in-order fragment; @@ -320,10 +333,13 @@ run immediately. `code_change_metrics` counter. `sync_batch.v4` returns `sync_ack.v4`, which additionally adds the `quota_cycle_contributions` counter. +`sync_batch.v5` returns `sync_ack.v5`, which additionally adds the +`account_plan_observations` and `account_evidence_summaries` counters. Collectors require the acknowledgement version to match the submitted batch version exactly; a v1 acknowledgement cannot successfully acknowledge a v2 -batch, a v2 acknowledgement cannot acknowledge a v3 batch, and a v3 -acknowledgement cannot acknowledge a v4 batch. +batch, a v2 acknowledgement cannot acknowledge a v3 batch, a v3 +acknowledgement cannot acknowledge a v4 batch, and a v4 acknowledgement cannot +acknowledge a v5 batch. The current loopback daemon returns this shape and reports duplicate events when the existing store already has the semantic event. Source, account, @@ -393,8 +409,8 @@ metadata. Full HTTP rollup syncs send their authoritative snapshot as the final logical chunk. The marker lists all current source, provider-account, -source-account-assignment, subscription, summary, code-change metric, and -quota-cycle contribution IDs, +source-account-assignment, subscription, summary, code-change metric, +quota-cycle contribution, account-plan-observation, and evidence-summary IDs, including empty lists. The backend tracks ownership per authenticated device and keeps device-local IDs separate from server-canonical IDs so account alias reconciliation cannot delete @@ -408,8 +424,7 @@ snapshot are retained through their active ownership mapping; omitted unowned rows are pruned. Legacy rows from other devices, and canonical rows still owned by any device, are preserved. -The HTTP sink parses `sync_ack.v1`, `sync_ack.v2`, `sync_ack.v3`, and -`sync_ack.v4` before +The HTTP sink parses `sync_ack.v1` through `sync_ack.v5` before updating local state. File and stdout sinks update state after their local write succeeds. @@ -446,7 +461,8 @@ The Worker rejects raw event cloud sync by default and accepts sanitized daily summary rollups plus metadata, along with hosted task snapshots and hosted task verification actions for `sync_batch.v2` and later, plus code-change metrics for `sync_batch.v3` and later, plus quota-cycle contributions for -`sync_batch.v4`. The collector now prepares those +`sync_batch.v4` and later, plus privacy-safe account-plan evidence for +`sync_batch.v5`. The collector now prepares those daily rollups before HTTP sync, so a normal Cloudflare sync can populate the dashboard without shipping raw events. Repeated batches are idempotent by stable IDs. @@ -459,7 +475,8 @@ HTTP chunking is a transport concern and must not change canonical hosted state. Account aliases are persisted per user and device in `sync_entity_owners`. Every later assignment, subscription, and summary chunk resolves its device-local account ID through that mapping, even when the chunk -contains no account records. When a newly observed alias matches historical +contains no account records. Account-plan observations and evidence summaries +use the same mapping. When a newly observed alias matches historical rows from that device, the backend repairs their indexed account ID and JSON payload in the same D1 transaction. The repair first discovers affected daily months and period rows, then rebuilds their monthly rollups and the all-time diff --git a/docs/task-collection.md b/docs/task-collection.md index dcb10a4..5b62ac2 100644 --- a/docs/task-collection.md +++ b/docs/task-collection.md @@ -3,7 +3,7 @@ `statsai` can extract local task spans during `scan`, rebuild derived work items in SQLite, and let you verify or correct those work items from the CLI. Older usage-only clients still emit `sync_batch.v1`. Hosted task sync was introduced -with `sync_batch.v2`; current `sync_batch.v3` clients retain the same task bucket +with `sync_batch.v2`; current `sync_batch.v4` clients retain the same task bucket snapshot and task verification fields. ## Normal Loop From 2e4851c0ed7fb1b7df19fa90f099cd40d1ad452a Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Tue, 25 Aug 2026 15:38:07 +0300 Subject: [PATCH 02/13] Fix eight defects in the account-plan evidence path Attribution: - A conversation bound to two accounts no longer discards a manual assignment. Clearing the account left `account_identity_source` still naming `UserConfigured` against an account that was gone. - Reset history no longer truncates a source-wide assignment. The truncating loop accepted any strong observation while only an auth reload could reopen an interval, so one conversation-scoped entry naming another account unattributed the rest of the source for good. Evidence integrity: - Telemetry identity is read only from the structured attribute prefix, and only when the body names its event and its account exactly once. A prompt quoting `event.name` and `user.account_id` could otherwise mint a binding for an account this device never used. - A login is identified by what it says and when, not by its file and line, so rotating `codex-login.log` no longer duplicates the login history that these append-only rows keep forever. Durability: - The legacy plan conversion runs inside `Store::migrate`, so one unparsable subscription or account payload rolled the transaction back before the completion flag was written and failed every later open. It now skips rows it cannot read; they could not have been converted. Cost: - Quota rows carrying the same plan collapse into one observation per run instead of one per raw reading. Ten thousand rows saying "Pro" were ten thousand ledger rows, as many sync queries, and a snapshot split across fifty fragments. - Conversation reattribution reads only its own source's events and returns early when the source has no bindings, instead of deserializing every row of `usage_events` on every scan. Docs: - The account-plan privacy paragraph no longer splits the stripped-field list into two disconnected lists. --- crates/statsai-adapters/src/lib.rs | 232 ++++++++++++++++++----- crates/statsai-core/src/account_plan.rs | 17 ++ crates/statsai-store/src/account_plan.rs | 208 ++++++++++++++++---- crates/statsai-store/src/lib.rs | 232 +++++++++++++++++++++++ crates/statsai-store/src/quota.rs | 114 +++++++++++ 5 files changed, 720 insertions(+), 83 deletions(-) diff --git a/crates/statsai-adapters/src/lib.rs b/crates/statsai-adapters/src/lib.rs index 03ea70a..cb50789 100644 --- a/crates/statsai-adapters/src/lib.rs +++ b/crates/statsai-adapters/src/lib.rs @@ -8069,6 +8069,13 @@ fn collect_codex_telemetry_evidence( let event_name = Regex::new(r#"(?:^|\s)event\.name\s*=\s*\"([a-z0-9._-]+)\""#)?; let auth_reload = Regex::new(r#"(?i)^\s*Reloading auth for account\s+([a-z0-9_:\-]{3,256})(?:\s|$)"#)?; + // Anything a user typed can reach this body verbatim inside one of these + // fields. Identity attributes are only read from the structured prefix that + // precedes the first of them, so a prompt quoting `user.account_id=...` + // cannot mint a binding for an account this device never used. + let free_text_attribute = Regex::new( + r#"(?i)(?:^|\s)(?:prompt|message|text|content|body|input|output|arguments|args|command|reasoning|response|error)\s*[:=]"#, + )?; for database_path in [ root.join("logs_2.sqlite"), @@ -8175,13 +8182,24 @@ fn collect_codex_telemetry_evidence( else { continue; }; + // Only the structured attribute prefix is trusted; everything from + // the first free-text field onward is user content. + let attributes = free_text_attribute + .find(&body) + .map_or(body.as_str(), |free_text| &body[..free_text.start()]); + // A body naming the event twice is not a body this parser + // understands. Reading only the first match let a second, injected + // `event.name` sit unnoticed beside the real one. + let mut event_names = event_name + .captures_iter(attributes) + .filter_map(|captures| captures.get(1).map(|value| value.as_str())); + let declared_event_name = event_names.next(); let structured_identity_event = matches!( target.as_str(), "codex_otel.log_only" | "codex_otel.trace_safe" - ) && event_name - .captures(&body) - .and_then(|captures| captures.get(1)) - .is_some_and(|value| value.as_str() == "codex.conversation_starts"); + ) && declared_event_name + == Some("codex.conversation_starts") + && event_names.next().is_none(); let reload_account_id = target .starts_with("codex_core::auth") .then(|| { @@ -8191,50 +8209,33 @@ fn collect_codex_telemetry_evidence( .map(|value| value.as_str().to_string()) }) .flatten(); + // A repeated attribute is ambiguous evidence, not two facts, so an + // event that states an identity twice states it for nobody. + let sole_attribute = |pattern: &Regex| -> Option { + let mut matches = pattern + .captures_iter(attributes) + .filter_map(|captures| captures.get(1).map(|value| value.as_str().to_string())); + let first = matches.next()?; + matches.next().is_none().then_some(first) + }; let provider_user_id = reload_account_id.clone().or_else(|| { - structured_identity_event - .then(|| { - account_attribute - .captures(&body) - .and_then(|captures| captures.get(1)) - .map(|value| value.as_str().to_string()) - }) - .flatten() + structured_identity_event.then(|| sole_attribute(&account_attribute))? }); let email = structured_identity_event - .then(|| { - email_attribute - .captures(&body) - .and_then(|captures| captures.get(1)) - .map(|value| normalize_email(value.as_str())) - }) - .flatten(); + .then(|| sole_attribute(&email_attribute)) + .flatten() + .map(|value| normalize_email(&value)); if provider_user_id.is_none() && email.is_none() { continue; } let conversation_id = structured_identity_event - .then(|| { - conversation_attribute - .captures(&body) - .and_then(|captures| captures.get(1)) - .map(|value| value.as_str().to_string()) - }) + .then(|| sole_attribute(&conversation_attribute)) .flatten(); let auth_mode = structured_identity_event - .then(|| { - auth_mode_attribute - .captures(&body) - .and_then(|captures| captures.get(1)) - .map(|value| value.as_str().to_string()) - }) + .then(|| sole_attribute(&auth_mode_attribute)) .flatten(); let application_version = structured_identity_event - .then(|| { - app_version_attribute - .captures(&body) - .and_then(|captures| captures.get(1)) - .map(|value| value.as_str().to_string()) - }) + .then(|| sole_attribute(&app_version_attribute)) .flatten(); let provider_account_id = provider_account_id_from_identity( CODEX_PROVIDER, @@ -8450,17 +8451,12 @@ fn collect_codex_login_evidence( let artifact_path_hash = hash_text(&canonical_display(&login_path)); let mut reader = BufReader::new(file); let mut line = Vec::new(); - let mut line_number = 0u64; loop { match read_bounded_jsonl_line(&mut reader, &mut line, 64 * 1024)? { BoundedLineRead::Eof => break, - BoundedLineRead::Oversized => { - line_number += 1; - continue; - } + BoundedLineRead::Oversized => continue, BoundedLineRead::Complete => {} } - line_number += 1; let Ok(text) = std::str::from_utf8(&line) else { continue; }; @@ -8483,10 +8479,15 @@ fn collect_codex_login_evidence( continue; }; let observed_at = observed_at.with_timezone(&Utc); + // Identify the login by what it says and when, never by where it + // currently sits. Hashing the file and line number meant the same + // login was a new observation the moment `codex-login.log` rotated + // to `codex-login.log.1`, and these rows are append-only, so every + // rotation permanently doubled the login history. let record_fingerprint = hash_text(&format!( - "codex-login-success.v1:{}:{line_number}:{}", - artifact_path_hash, - observed_at.to_rfc3339() + "codex-login-success.v1:{}:{}", + observed_at.to_rfc3339(), + text.trim() )); scan.identity_observations .push(AccountIdentityObservationV1 { @@ -13226,6 +13227,145 @@ mod tests { .expect("unlock database"); } + #[test] + fn codex_telemetry_identity_ignores_attributes_quoted_inside_user_text() { + let dir = tempfile::tempdir().expect("tempdir"); + let connection = Connection::open(dir.path().join("logs_2.sqlite")).expect("logs database"); + connection + .execute_batch( + r#" + CREATE TABLE logs ( + id INTEGER PRIMARY KEY, + ts INTEGER, + ts_nanos INTEGER, + target TEXT NOT NULL, + feedback_log_body TEXT + ); + "#, + ) + .expect("logs schema"); + for (row_id, body) in [ + // The injected attributes lead the body, so reading the first match + // anywhere accepted them as the event's own identity. + ( + 1_i64, + "prompt=\"please run event.name=\\\"codex.conversation_starts\\\" user.account_id=acct-attacker user.email=attacker@example.test\" event.name=\"codex.user_prompt\"", + ), + // A genuine event whose free text repeats the marker afterwards. + ( + 2, + "event.name=\"codex.conversation_starts\" user.account_id=acct-real prompt=\"see event.name=\\\"codex.conversation_starts\\\" user.account_id=acct-attacker\"", + ), + // Two structured identities in one body name nobody. + ( + 3, + "event.name=\"codex.conversation_starts\" user.account_id=acct-one user.account_id=acct-two", + ), + ] { + connection + .execute( + "INSERT INTO logs (id, ts, ts_nanos, target, feedback_log_body) VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params![ + row_id, + 1_787_227_200_i64 + row_id, + 0_i64, + "codex_otel.log_only", + body + ], + ) + .expect("telemetry row"); + } + drop(connection); + + let source = SourceLocation::local_adapter( + CODEX_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + let evidence = CodexAdapter + .collect_account_evidence(&source, &[]) + .expect("account evidence"); + + let identified = evidence + .identity_observations + .iter() + .filter(|item| item.evidence_kind == AccountEvidenceKind::TelemetryIdentity) + .collect::>(); + assert_eq!( + identified.len(), + 1, + "only the genuine structured attribute prefix identifies an account" + ); + let expected = provider_account_id_from_identity(CODEX_PROVIDER, Some("acct-real"), None) + .expect("account id"); + assert_eq!(identified[0].provider_account_id.as_ref(), Some(&expected)); + let attacker = + provider_account_id_from_identity(CODEX_PROVIDER, Some("acct-attacker"), None) + .expect("account id"); + assert!( + evidence + .conversation_bindings + .iter() + .all(|binding| binding.provider_account_id != attacker), + "prompt text must never bind a conversation to an unused account" + ); + assert!(evidence + .accounts + .iter() + .all(|account| account.provider_user_id.as_deref() != Some("acct-attacker"))); + } + + #[test] + fn codex_login_evidence_survives_log_rotation_without_duplicating() { + let dir = tempfile::tempdir().expect("tempdir"); + let log_directory = dir.path().join("log"); + std::fs::create_dir_all(&log_directory).expect("log directory"); + let entry = "2026-08-20T12:00:00Z successfully logged in\n"; + std::fs::write(log_directory.join("codex-login.log"), entry).expect("login log"); + let source = SourceLocation::local_adapter( + CODEX_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + + let before = CodexAdapter + .collect_account_evidence(&source, &[]) + .expect("evidence before rotation"); + let before_ids = before + .identity_observations + .iter() + .filter(|item| item.evidence_kind == AccountEvidenceKind::LoginSuccess) + .map(|item| item.observation_id.clone()) + .collect::>(); + assert_eq!(before_ids.len(), 1); + + // The same login, now one generation older, plus a fresh empty log. + std::fs::rename( + log_directory.join("codex-login.log"), + log_directory.join("codex-login.log.1"), + ) + .expect("rotate login log"); + std::fs::write(log_directory.join("codex-login.log"), "").expect("fresh login log"); + + let after = CodexAdapter + .collect_account_evidence(&source, &[]) + .expect("evidence after rotation"); + let after_ids = after + .identity_observations + .iter() + .filter(|item| item.evidence_kind == AccountEvidenceKind::LoginSuccess) + .map(|item| item.observation_id.clone()) + .collect::>(); + assert_eq!( + after_ids, before_ids, + "rotation moves a login between files; it is not a second login" + ); + } + #[test] fn codex_probe_verified_source_state_uses_parent_auth_for_sessions_path() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/crates/statsai-core/src/account_plan.rs b/crates/statsai-core/src/account_plan.rs index 6a0c7d5..18912a3 100644 --- a/crates/statsai-core/src/account_plan.rs +++ b/crates/statsai-core/src/account_plan.rs @@ -33,6 +33,23 @@ impl AccountEvidenceKind { Self::AuthSnapshot | Self::TelemetryIdentity | Self::AuthReload | Self::ResetHistory ) } + + /// Whether this evidence says the *source* itself changed account at that moment. + /// + /// `ResetHistory` is deliberately excluded. It is replayed from a persisted + /// state file, is scoped to one conversation and turn, and describes a + /// rate-limit reset rather than an authentication state, so it can name an + /// account the source was never signed in as. Only kinds that can also + /// reopen an interval belong here: ending an interval on evidence that + /// cannot restart one leaves the rest of the source's history permanently + /// unattributed, with no later scan able to repair it. + #[must_use] + pub const fn ends_source_attribution(self) -> bool { + matches!( + self, + Self::AuthSnapshot | Self::TelemetryIdentity | Self::AuthReload + ) + } } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] diff --git a/crates/statsai-store/src/account_plan.rs b/crates/statsai-store/src/account_plan.rs index d9806d3..da7dccc 100644 --- a/crates/statsai-store/src/account_plan.rs +++ b/crates/statsai-store/src/account_plan.rs @@ -262,6 +262,48 @@ impl Store { }) } + /// Reads the subscriptions this one-shot conversion can act on, skipping rows it cannot parse. + /// + /// This runs inside `Store::migrate`, so an error here fails `Store::open` for good: the + /// transaction rolls back, the completion flag is never written, and every later launch + /// retries the same row and fails again. One payload written by a newer build, or corrupted + /// on disk, is not a reason to make the store unopenable — and an unparsable record could + /// not have been converted anyway. It stays where it is, still visible as a legacy row. + fn subscriptions_for_legacy_plan_migration(&self) -> Result> { + let mut statement = self.conn.prepare( + "SELECT payload, provider_account_id FROM subscriptions ORDER BY provider, subscription_id", + )?; + let rows = statement.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)) + })?; + let mut subscriptions = Vec::new(); + for row in rows { + let (payload, provider_account_id) = row?; + if let Ok(subscription) = + super::deserialize_subscription_payload(&payload, provider_account_id.as_deref()) + { + subscriptions.push(subscription); + } + } + Ok(subscriptions) + } + + /// The account counterpart of [`Self::subscriptions_for_legacy_plan_migration`], and skipping + /// for the same reason. + fn accounts_for_legacy_plan_migration(&self) -> Result> { + let mut statement = self.conn.prepare( + "SELECT payload FROM provider_accounts ORDER BY provider, provider_account_id", + )?; + let rows = statement.query_map([], |row| row.get::<_, String>(0))?; + let mut accounts = Vec::new(); + for row in rows { + if let Ok(account) = serde_json::from_str(&row?) { + accounts.push(account); + } + } + Ok(accounts) + } + /// Converts historical Codex subscriptions and account-level plan fields synthesized from /// local authentication into provider-plan evidence. User-entered billing records are /// deliberately left untouched. @@ -270,7 +312,7 @@ impl Store { /// discard the legacy record. The deterministic observation ID makes this safe to repeat. pub fn migrate_legacy_codex_local_auth_subscriptions_to_plan_evidence(&self) -> Result { self.with_immediate_transaction(|| { - let subscriptions = self.list_subscriptions()?; + let subscriptions = self.subscriptions_for_legacy_plan_migration()?; let mut converted = 0u64; for subscription in subscriptions.into_iter().filter(|subscription| { subscription.provider.eq_ignore_ascii_case("codex") @@ -335,7 +377,7 @@ impl Store { }) .collect::>(); for mut account in self - .list_accounts()? + .accounts_for_legacy_plan_migration()? .into_iter() .filter(|account| account.provider.eq_ignore_ascii_case("codex")) { @@ -422,8 +464,12 @@ impl Store { let mut changed = 0u64; // A contradictory strong point invalidates generated continuity from that point. The - // previous account is not resumed unless a later explicit boundary supports it. - for observation in &strong { + // previous account is not resumed unless a later explicit boundary supports it, so + // only evidence that could supply such a boundary is allowed to end an interval. + for observation in strong + .iter() + .filter(|observation| observation.evidence_kind.ends_source_attribution()) + { let account_id = observation .provider_account_id .as_ref() @@ -556,13 +602,28 @@ impl Store { /// Convert an attributed quota status into plan evidence. A plan label by itself never /// identifies an account, so records without an unambiguous source assignment are skipped. + /// + /// Quota rows arrive one per provider response, and a plan label barely ever changes between + /// them: ten thousand rows saying "Pro" are one fact, not ten thousand. Keying each row's own + /// `observed_at` made them ten thousand ledger rows, ten thousand `entity_requires_sync` + /// queries, and a snapshot split across fifty-odd fragments. Consecutive rows carrying the + /// same label for the same account therefore collapse into one observation spanning the run, + /// timed at its most recent evidence and identified by its first, so that history stays put + /// and only the open run re-syncs as it grows. pub fn upsert_quota_plan_observations( &self, records: &[QuotaObservationRecordV1], ) -> Result { + struct PlanRun<'a> { + quota: &'a statsai_core::QuotaObservationV1, + provider_account_id: ProviderAccountId, + raw_plan_name: &'a str, + started_at: chrono::DateTime, + } + let mut assignments_by_source = HashMap::new(); - let mut observations = Vec::new(); - let mut seen = HashSet::new(); + let mut attributed: Vec<(ProviderAccountId, &str, &statsai_core::QuotaObservationV1)> = + Vec::new(); for record in records { let quota = &record.observation; let Some(raw_plan_name) = quota @@ -591,35 +652,74 @@ impl Store { let Some(provider_account_id) = provider_account_id else { continue; }; - let observation_id = account_plan_observation_id( - "a.source_id, - Some(&provider_account_id), - raw_plan_name, - quota.observed_at, - AccountEvidenceKind::QuotaStatus, - ); - if !seen.insert(observation_id.clone()) { + attributed.push((provider_account_id, raw_plan_name, quota)); + } + // Records reach here in scan order, which is not observation order once a + // rotated file or a re-import interleaves them. A run is only meaningful + // along the timeline, so sort before collapsing. + attributed.sort_by(|left, right| { + left.2 + .source_id + .0 + .cmp(&right.2.source_id.0) + .then_with(|| left.0 .0.cmp(&right.0 .0)) + .then_with(|| left.2.observed_at.cmp(&right.2.observed_at)) + .then_with(|| left.2.observation_id.cmp(&right.2.observation_id)) + }); + + let mut runs: Vec> = Vec::new(); + for (provider_account_id, raw_plan_name, quota) in attributed { + let continues_run = runs.last().is_some_and(|run| { + run.quota.source_id == quota.source_id + && run.provider_account_id == provider_account_id + && run.raw_plan_name.eq_ignore_ascii_case(raw_plan_name) + }); + if continues_run { + let run = runs.last_mut().expect("checked above"); + run.quota = quota; + run.raw_plan_name = raw_plan_name; continue; } - observations.push(AccountPlanObservationV1 { - schema_version: ACCOUNT_PLAN_OBSERVATION_SCHEMA_VERSION.to_string(), - observation_id, - provider: quota.provider.clone(), - source_id: quota.source_id.clone(), - provider_account_id: Some(provider_account_id), - raw_plan_name: raw_plan_name.to_string(), - plan_name: normalize_plan_name(raw_plan_name), - observed_at: quota.observed_at, - active_from: None, - active_until: None, - is_current_snapshot: false, - evidence_kind: AccountEvidenceKind::QuotaStatus, - confidence: Confidence::High, - parser_version: "quota-plan-evidence.v1".to_string(), - artifact_path_hash: quota.source_file_path_hash.clone(), - record_fingerprint: quota.semantic_fingerprint.clone(), + runs.push(PlanRun { + quota, + provider_account_id, + raw_plan_name, + started_at: quota.observed_at, }); } + + let observations = runs + .into_iter() + .map(|run| { + let quota = run.quota; + AccountPlanObservationV1 { + schema_version: ACCOUNT_PLAN_OBSERVATION_SCHEMA_VERSION.to_string(), + // Identified by where the run began, so extending it does not + // mint a new row and retire the old one on every scan. + observation_id: account_plan_observation_id( + "a.source_id, + Some(&run.provider_account_id), + run.raw_plan_name, + run.started_at, + AccountEvidenceKind::QuotaStatus, + ), + provider: quota.provider.clone(), + source_id: quota.source_id.clone(), + provider_account_id: Some(run.provider_account_id), + raw_plan_name: run.raw_plan_name.to_string(), + plan_name: normalize_plan_name(run.raw_plan_name), + observed_at: quota.observed_at, + active_from: None, + active_until: None, + is_current_snapshot: false, + evidence_kind: AccountEvidenceKind::QuotaStatus, + confidence: Confidence::High, + parser_version: "quota-plan-evidence.v1".to_string(), + artifact_path_hash: quota.source_file_path_hash.clone(), + record_fingerprint: quota.semantic_fingerprint.clone(), + } + }) + .collect::>(); self.upsert_account_plan_observations(&observations) } @@ -829,7 +929,32 @@ impl Store { continue; }; if accounts.len() != 1 { - event.provider_account_id = None; + // Conflicting bindings are weaker evidence than a person telling + // us who owns this source. Clearing the account here dropped a + // `UserConfigured` assignment and left `account_identity_source` + // still naming it, so the event claimed a manual attribution to + // an account that was no longer on it. Leave the event as it is + // and let the conflict stay visible in the evidence summary. + if !matches!( + event + .parse_evidence + .as_ref() + .map(|evidence| &evidence.account_identity_source), + Some(IdentitySource::UserConfigured) + ) { + event.provider_account_id = None; + } + continue; + } + // A manual assignment also outranks a single agreeing binding: it is + // the only source the user can correct by hand. + if matches!( + event + .parse_evidence + .as_ref() + .map(|evidence| &evidence.account_identity_source), + Some(IdentitySource::UserConfigured) + ) { continue; } event.provider_account_id = accounts.iter().next().map(|value| (*value).clone()); @@ -841,11 +966,20 @@ impl Store { } pub fn reattribute_conversation_bound_events(&self, source_id: &SourceId) -> Result { - let mut events = self - .events()? - .into_iter() - .filter(|event| event.source_id == *source_id) - .collect::>(); + // Nothing to reattribute without bindings, and this is the common case: + // every scan of every Auto-verified source reached here. Checking first + // avoids loading and deserializing the source's events for no reason. + if self + .conversation_account_bindings(Some(source_id))? + .is_empty() + { + return Ok(0); + } + // `events()` reads every row in `usage_events` and deserializes each + // payload before this filter sees it. On a store with hundreds of + // thousands of events that is the whole table parsed to keep one + // source's share; `events_for_source` pushes the same filter into SQL. + let mut events = self.events_for_source(source_id)?; let previous_accounts = events .iter() .map(|event| event.provider_account_id.clone()) diff --git a/crates/statsai-store/src/lib.rs b/crates/statsai-store/src/lib.rs index 7f2b324..3a20fb9 100644 --- a/crates/statsai-store/src/lib.rs +++ b/crates/statsai-store/src/lib.rs @@ -7007,6 +7007,65 @@ mod tests { ); } + #[test] + fn an_unreadable_legacy_payload_does_not_make_the_store_unopenable() { + let path = tempfile::tempdir() + .expect("tempdir") + .keep() + .join("store.db"); + { + let store = Store::open(&path).expect("initial store"); + store + .conn + .execute( + "INSERT INTO subscriptions + (subscription_id, provider, provider_account_id, payload) + VALUES (?1, ?2, ?3, ?4)", + params![ + "corrupt-subscription", + "codex", + "codex:corrupt", + "{ this is not json", + ], + ) + .expect("corrupt subscription row"); + store + .conn + .execute( + "INSERT INTO provider_accounts + (provider_account_id, provider, payload, updated_at) + VALUES (?1, ?2, ?3, ?4)", + params![ + "codex:corrupt-account", + "codex", + "{ also not json", + Utc::now().to_rfc3339(), + ], + ) + .expect("corrupt account row"); + store + .conn + .execute( + "DELETE FROM local_metadata WHERE key = ?1", + params![LEGACY_CODEX_PLAN_CONVERSION_METADATA_KEY], + ) + .expect("clear conversion flag"); + } + + // The conversion runs inside `migrate`, so a hard error here would roll + // back before the completion flag is written and fail every later open. + let reopened = Store::open(&path).expect("a corrupt legacy row must not brick the store"); + assert_eq!( + reopened + .metadata_value(LEGACY_CODEX_PLAN_CONVERSION_METADATA_KEY) + .expect("conversion flag") + .as_deref(), + Some("1"), + "the one-shot conversion must be recorded as done, not retried forever" + ); + Store::open(&path).expect("a second open still succeeds"); + } + #[test] fn migrates_legacy_codex_account_plan_without_subscription() { let store = Store::in_memory().expect("store"); @@ -10895,6 +10954,179 @@ mod tests { assert!(epoch_floor.is_empty()); } + #[test] + fn conflicting_bindings_keep_a_manual_account_assignment() { + let store = Store::in_memory().expect("store"); + let now = Utc::now(); + let source = statsai_core::SourceLocation::local_adapter( + "codex", + "test", + "0", + std::path::Path::new("/tmp/binding-conflict"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let manual = ProviderAccountId("account-manual".to_string()); + + // The same conversation is bound to two different accounts. + for (index, account) in ["account-one", "account-two"].iter().enumerate() { + store + .upsert_conversation_account_bindings(&[ + statsai_core::ConversationAccountBindingV1 { + schema_version: statsai_core::CONVERSATION_ACCOUNT_BINDING_SCHEMA_VERSION + .to_string(), + binding_id: format!("binding-{index}"), + provider: "codex".to_string(), + source_id: source.source_id.clone(), + provider_account_id: ProviderAccountId((*account).to_string()), + conversation_id_hash: "same-session".to_string(), + turn_id_hash: None, + observed_at: now, + evidence_kind: statsai_core::AccountEvidenceKind::TelemetryIdentity, + confidence: Confidence::High, + }, + ]) + .expect("binding"); + } + + let mut manual_event = test_store_event(&source, now, "manual-record"); + manual_event.provider_account_id = Some(manual.clone()); + manual_event.parse_evidence = Some(ParseEvidence { + event_key_version: "test".to_string(), + source_file_path_hash: None, + source_line_number: None, + source_record_id: None, + model_inferred: false, + timestamp_inferred: false, + account_identity_source: IdentitySource::UserConfigured, + }); + let mut derived_event = test_store_event(&source, now, "derived-record"); + derived_event.provider_account_id = Some(ProviderAccountId("account-one".to_string())); + derived_event.parse_evidence = Some(ParseEvidence { + account_identity_source: IdentitySource::LocalAuth, + ..manual_event + .parse_evidence + .clone() + .expect("evidence template") + }); + + let mut events = vec![manual_event, derived_event]; + store + .apply_conversation_account_bindings(&source.source_id, &mut events) + .expect("apply bindings"); + + assert_eq!( + events[0].provider_account_id.as_ref(), + Some(&manual), + "a conflict between derived bindings must not discard a manual assignment" + ); + assert_eq!( + events[0] + .parse_evidence + .as_ref() + .map(|evidence| &evidence.account_identity_source), + Some(&IdentitySource::UserConfigured), + "the recorded identity source must still describe the account on the event" + ); + assert_eq!( + events[1].provider_account_id, None, + "a derived attribution is still cleared by a genuine conflict" + ); + } + + #[test] + fn reset_history_alone_does_not_truncate_a_source_assignment() { + let store = Store::in_memory().expect("store"); + let started_at = Utc + .with_ymd_and_hms(2026, 6, 1, 0, 0, 0) + .single() + .expect("started at"); + let source = statsai_core::SourceLocation::local_adapter( + "codex", + "test", + "0", + std::path::Path::new("/tmp/reset-history-truncation"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let bound = ProviderAccountId("account-bound".to_string()); + let assignment = statsai_core::SourceAccountAssignment { + schema_version: statsai_core::SOURCE_ACCOUNT_ASSIGNMENT_SCHEMA_VERSION.to_string(), + assignment_id: statsai_core::SourceAccountAssignmentId("assignment-1".to_string()), + source_id: source.source_id.clone(), + provider: "codex".to_string(), + provider_account_id: bound.clone(), + started_at, + ended_at: None, + record_source: IdentitySource::LocalAuth, + verified_at: Some(started_at), + created_at: started_at, + updated_at: started_at, + }; + store + .upsert_source_account_assignment(&assignment) + .expect("assignment"); + + // An auth snapshot corroborates the assignment, then a single + // conversation-scoped reset-history entry names a different account. + for (index, (kind, account, offset_days)) in [ + ( + statsai_core::AccountEvidenceKind::AuthSnapshot, + "account-bound", + 1_i64, + ), + ( + statsai_core::AccountEvidenceKind::ResetHistory, + "account-other", + 2, + ), + ] + .into_iter() + .enumerate() + { + store + .upsert_account_identity_observations(&[ + statsai_core::AccountIdentityObservationV1 { + schema_version: statsai_core::ACCOUNT_IDENTITY_OBSERVATION_SCHEMA_VERSION + .to_string(), + observation_id: format!("identity-{index}"), + provider: "codex".to_string(), + source_id: source.source_id.clone(), + provider_account_id: Some(ProviderAccountId(account.to_string())), + provider_user_id_hash: None, + email_hash: None, + conversation_id_hash: Some("f".repeat(64)), + turn_id_hash: Some("a".repeat(64)), + observed_at: started_at + chrono::Duration::days(offset_days), + evidence_kind: kind, + confidence: Confidence::High, + auth_mode: None, + application_version: None, + parser_version: "test.v1".to_string(), + artifact_kind: "test".to_string(), + artifact_path_hash: "c".repeat(64), + record_fingerprint: format!("{index}").repeat(64), + }, + ]) + .expect("identity observation"); + } + + store + .reconcile_source_account_evidence_assignments(&source.source_id) + .expect("reconcile"); + + let assignments = store + .list_source_account_assignments_for_source(&source.source_id) + .expect("assignments"); + assert_eq!(assignments.len(), 1); + assert_eq!(assignments[0].provider_account_id, bound); + assert_eq!( + assignments[0].ended_at, None, + "a per-conversation reset-history entry cannot end a source-wide assignment, \ + because nothing downstream is able to reopen one" + ); + } + fn test_store_event( source: &statsai_core::SourceLocation, now: chrono::DateTime, diff --git a/crates/statsai-store/src/quota.rs b/crates/statsai-store/src/quota.rs index 8cd092b..e9f9442 100644 --- a/crates/statsai-store/src/quota.rs +++ b/crates/statsai-store/src/quota.rs @@ -1729,6 +1729,120 @@ mod tests { assert_ne!(corrected[0].observation_id, initial[0].observation_id); } + #[test] + fn many_quota_rows_naming_one_plan_collapse_into_one_observation() { + let store = Store::in_memory().expect("store"); + let start = DateTime::from_timestamp(1_787_227_200, 0).expect("start"); + let (source_id, account_id) = assigned_source(&store, start - Duration::days(1)); + + // Two hundred provider responses across two plans: "pro", then "team", + // then "pro" again after a downgrade. + let mut records = Vec::new(); + for index in 0..200i64 { + let observed_at = start + Duration::minutes(index); + let mut record = sample_record( + source_id.clone(), + &format!("quota-run-{index}"), + &format!("quota-run-fingerprint-{index}"), + observed_at, + (observed_at + Duration::days(7)).timestamp(), + "primary", + 10_080, + 25.0, + ); + record.observation.status.plan_type = Some( + match index { + ..=99 => "pro", + 100..=149 => "team", + _ => "pro", + } + .to_string(), + ); + records.push(record); + } + store + .upsert_quota_observations(&records) + .expect("quota observations"); + store + .rebuild_quota_plan_observations_for_source(&source_id) + .expect("plan rebuild"); + + let observations = store.account_plan_observations().expect("plan evidence"); + assert_eq!( + observations.len(), + 3, + "one observation per run of an unchanged plan, not one per quota row" + ); + let mut runs = observations + .iter() + .map(|observation| { + ( + observation.raw_plan_name.as_str(), + observation.observed_at, + observation.provider_account_id.clone(), + ) + }) + .collect::>(); + runs.sort_by_key(|run| run.1); + assert_eq!(runs[0].0, "pro"); + assert_eq!(runs[0].1, start + Duration::minutes(99)); + assert_eq!(runs[1].0, "team"); + assert_eq!(runs[1].1, start + Duration::minutes(149)); + assert_eq!(runs[2].0, "pro"); + assert_eq!(runs[2].1, start + Duration::minutes(199)); + assert!(runs.iter().all(|run| run.2.as_ref() == Some(&account_id))); + + // Rebuilding without new data must not churn the ledger. + let ids_before = observations + .iter() + .map(|observation| observation.observation_id.clone()) + .collect::>(); + store + .rebuild_quota_plan_observations_for_source(&source_id) + .expect("repeat plan rebuild"); + let ids_after = store + .account_plan_observations() + .expect("plan evidence") + .iter() + .map(|observation| observation.observation_id.clone()) + .collect::>(); + assert_eq!(ids_after, ids_before); + + // Extending the open run keeps its identity, so history stays put and + // only the newest observation re-syncs. + let extension = { + let observed_at = start + Duration::minutes(200); + let mut record = sample_record( + source_id.clone(), + "quota-run-200", + "quota-run-fingerprint-200", + observed_at, + (observed_at + Duration::days(7)).timestamp(), + "primary", + 10_080, + 26.0, + ); + record.observation.status.plan_type = Some("pro".to_string()); + record + }; + store + .upsert_quota_observations(&[extension]) + .expect("extend run"); + store + .rebuild_quota_plan_observations_for_source(&source_id) + .expect("plan rebuild after extension"); + let extended = store.account_plan_observations().expect("plan evidence"); + assert_eq!(extended.len(), 3); + assert_eq!( + extended + .iter() + .map(|observation| observation.observation_id.clone()) + .collect::>(), + ids_before, + "extending a run must not mint a new observation and retire the old one" + ); + } + #[test] fn quota_plan_label_never_identifies_an_unassigned_account() { let store = Store::in_memory().expect("store"); From 72211f9bf1e67d8d41a155280948653ef150e2a6 Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Tue, 25 Aug 2026 16:44:51 +0300 Subject: [PATCH 03/13] Keep reset history out of auth-reload interval bounds The truncating loop already ignored it, but the loop that reopens an interval still searched all strong evidence for its confirmation and its end. A turn-scoped reset-history entry naming another account could therefore close the interval an auth reload had just opened, leaving the rest of the source unattributed with nothing able to reopen it. Also names sync_batch.v5 as the current schema in the README and the task collection notes. --- README.md | 2 +- crates/statsai-store/src/account_plan.rs | 14 ++++- crates/statsai-store/src/lib.rs | 80 ++++++++++++++++++++++++ docs/task-collection.md | 2 +- 4 files changed, 93 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6eb8431..809b920 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ dashboard. Raw usage events and complete archived conversation records stay local and are never included in hosted sync. StatsAI does not upload full prompts, full responses, or raw provider logs. When hosted task sync is explicitly enabled -with `statsai sync --include-tasks`, the current `sync_batch.v4` payload may +with `statsai sync --include-tasks`, the current `sync_batch.v5` payload may include bounded conversation-derived task titles, summary previews, and todo excerpts. You can inspect the exact sync contract in [`docs/sync-contract.md`](docs/sync-contract.md) and verify the resolved sync diff --git a/crates/statsai-store/src/account_plan.rs b/crates/statsai-store/src/account_plan.rs index da7dccc..238b50f 100644 --- a/crates/statsai-store/src/account_plan.rs +++ b/crates/statsai-store/src/account_plan.rs @@ -506,6 +506,10 @@ impl Store { // A reload is an explicit boundary, but only a later strong point for the same account // promotes it into a source interval. Current auth snapshots are handled by the // existing verified-auth reconciliation path. + // + // Both searches below read source-wide evidence only, for the same reason the + // truncation above does: a turn-scoped reset-history entry neither proves the source + // was signed in as that account nor, naming another, that it stopped being. for (index, boundary) in strong.iter().enumerate().filter(|(_, observation)| { observation.evidence_kind == AccountEvidenceKind::AuthReload }) { @@ -513,14 +517,18 @@ impl Store { .provider_account_id .as_ref() .expect("filtered account identity"); - let confirmation = strong[index + 1..].iter().find(|observation| { + let later_source_evidence = || { + strong[index + 1..] + .iter() + .filter(|observation| observation.evidence_kind.ends_source_attribution()) + }; + let confirmation = later_source_evidence().find(|observation| { observation.provider_account_id.as_ref() == Some(account_id) }); let Some(confirmation) = confirmation else { continue; }; - let ended_at = strong[index + 1..] - .iter() + let ended_at = later_source_evidence() .find(|observation| { observation.provider_account_id.as_ref() != Some(account_id) }) diff --git a/crates/statsai-store/src/lib.rs b/crates/statsai-store/src/lib.rs index 3a20fb9..6d2411e 100644 --- a/crates/statsai-store/src/lib.rs +++ b/crates/statsai-store/src/lib.rs @@ -11127,6 +11127,86 @@ mod tests { ); } + #[test] + fn reset_history_does_not_bound_a_reopened_auth_reload_interval() { + let store = Store::in_memory().expect("store"); + let base = Utc + .with_ymd_and_hms(2026, 6, 1, 0, 0, 0) + .single() + .expect("base"); + let source = statsai_core::SourceLocation::local_adapter( + "codex", + "test", + "0", + std::path::Path::new("/tmp/reload-interval-bounds"), + LocationOrigin::Configured, + ); + store.upsert_source(&source).expect("source"); + let reloaded = ProviderAccountId("account-reloaded".to_string()); + + // reload A, telemetry A, then a turn-scoped reset-history entry naming B. + for (index, (kind, account, offset_days)) in [ + ( + statsai_core::AccountEvidenceKind::AuthReload, + "account-reloaded", + 1_i64, + ), + ( + statsai_core::AccountEvidenceKind::TelemetryIdentity, + "account-reloaded", + 2, + ), + ( + statsai_core::AccountEvidenceKind::ResetHistory, + "account-other", + 3, + ), + ] + .into_iter() + .enumerate() + { + store + .upsert_account_identity_observations(&[ + statsai_core::AccountIdentityObservationV1 { + schema_version: statsai_core::ACCOUNT_IDENTITY_OBSERVATION_SCHEMA_VERSION + .to_string(), + observation_id: format!("reload-identity-{index}"), + provider: "codex".to_string(), + source_id: source.source_id.clone(), + provider_account_id: Some(ProviderAccountId(account.to_string())), + provider_user_id_hash: None, + email_hash: None, + conversation_id_hash: Some("f".repeat(64)), + turn_id_hash: Some("a".repeat(64)), + observed_at: base + chrono::Duration::days(offset_days), + evidence_kind: kind, + confidence: Confidence::High, + auth_mode: None, + application_version: None, + parser_version: "test.v1".to_string(), + artifact_kind: "test".to_string(), + artifact_path_hash: "c".repeat(64), + record_fingerprint: format!("{index}").repeat(64), + }, + ]) + .expect("identity observation"); + } + + store + .reconcile_source_account_evidence_assignments(&source.source_id) + .expect("reconcile"); + + let assignments = store + .list_source_account_assignments_for_source(&source.source_id) + .expect("assignments"); + assert_eq!(assignments.len(), 1); + assert_eq!(assignments[0].provider_account_id, reloaded); + assert_eq!( + assignments[0].ended_at, None, + "reset history must not close the interval an auth reload opened" + ); + } + fn test_store_event( source: &statsai_core::SourceLocation, now: chrono::DateTime, diff --git a/docs/task-collection.md b/docs/task-collection.md index 5b62ac2..77493c6 100644 --- a/docs/task-collection.md +++ b/docs/task-collection.md @@ -3,7 +3,7 @@ `statsai` can extract local task spans during `scan`, rebuild derived work items in SQLite, and let you verify or correct those work items from the CLI. Older usage-only clients still emit `sync_batch.v1`. Hosted task sync was introduced -with `sync_batch.v2`; current `sync_batch.v4` clients retain the same task bucket +with `sync_batch.v2`; current `sync_batch.v5` clients retain the same task bucket snapshot and task verification fields. ## Normal Loop From 7134344c0fd46c4ddea6885a2dfe2faeff70f1ba Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:19:31 +0300 Subject: [PATCH 04/13] Date the auth identity by the login, not the plan check `chatgpt_subscription_last_checked` moves when the embedded plan claims are revalidated, which is not when the account changed: signing out of one account and into another rewrites the account id and leaves that stamp where it was. Using it for the identity observation therefore claimed the source was already on the new account weeks before the login, and an `AuthSnapshot` ends a source's account interval, so the previous account lost every event in between. The identity now carries `auth_time`; the plan observation keeps the subscription check. Quota cycles also stop counting as rollup metadata. They have their own splitting path and the backend upserts the whole collection in a single statement, so counting them charged them twice against the D1 budget, made `has_non_quota_cycle_payload` true for a batch holding nothing else -- which returned the same chunk from every split and retried it forever -- and sent each contribution through both splitters. --- crates/statsai-adapters/src/lib.rs | 62 ++++++++++++- crates/statsai/src/main.rs | 143 ++++++++++++++++++++++++----- 2 files changed, 176 insertions(+), 29 deletions(-) diff --git a/crates/statsai-adapters/src/lib.rs b/crates/statsai-adapters/src/lib.rs index cb50789..4baefcc 100644 --- a/crates/statsai-adapters/src/lib.rs +++ b/crates/statsai-adapters/src/lib.rs @@ -7946,10 +7946,20 @@ fn collect_codex_auth_evidence( let Some(claims) = codex_auth_claims(root) else { return; }; - let observed_at = claims + // Two different facts live in one file and they are not observed together. + // `chatgpt_subscription_last_checked` moves when the plan claims are + // revalidated, which can be long before the current login: signing out of + // A and into B rewrites the account without refreshing it. Dating the + // identity by that stale stamp said "this source was B" at a moment it was + // still A, and `ends_source_attribution` turns an AuthSnapshot into a + // source-wide boundary, so A's interval was truncated back to the last + // subscription check and every event in between lost its account. + let authenticated_at = claims.authenticated_at.unwrap_or_else(Utc::now); + let plan_observed_at = claims .subscription_checked_at .or(claims.authenticated_at) - .unwrap_or_else(Utc::now); + .unwrap_or(authenticated_at); + let observed_at = authenticated_at; let provider_account_id = provider_account_id_from_identity( CODEX_PROVIDER, claims.provider_user_id.as_deref(), @@ -8020,7 +8030,7 @@ fn collect_codex_auth_evidence( &source.source_id, provider_account_id.as_ref(), &raw_plan_name, - observed_at, + plan_observed_at, AccountEvidenceKind::AuthSnapshot, ); scan.plan_observations.push(AccountPlanObservationV1 { @@ -8031,7 +8041,7 @@ fn collect_codex_auth_evidence( provider_account_id, plan_name: normalize_plan_name(&raw_plan_name), raw_plan_name, - observed_at, + observed_at: plan_observed_at, active_from: claims.active_from, active_until: claims.active_until, is_current_snapshot: true, @@ -12815,6 +12825,50 @@ mod tests { ); } + #[test] + fn codex_auth_identity_is_dated_by_the_login_not_the_subscription_check() { + // `auth_time` is 2026-06-10; the embedded subscription claims were last + // revalidated on 2026-05-01. Signing into a different account rewrites + // the account id without touching that older stamp, so dating the + // identity by it would claim this source was already `acct-b` five + // weeks before the login — and `AuthSnapshot` ends a source's account + // interval, so the previous account would lose those five weeks. + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::write( + dir.path().join("auth.json"), + serde_json::json!({ + "tokens": { + "id_token": "eyJhbGciOiJub25lIn0.eyJlbWFpbCI6InBlcnNvbkBleGFtcGxlLmNvbSIsImF1dGhfdGltZSI6MTc4MTA0OTYwMCwiaHR0cHM6Ly9hcGkub3BlbmFpLmNvbS9hdXRoIjp7ImNoYXRncHRfYWNjb3VudF9pZCI6ImFjY3QtYiIsImNoYXRncHRfcGxhbl90eXBlIjoicGx1cyIsImNoYXRncHRfc3Vic2NyaXB0aW9uX2xhc3RfY2hlY2tlZCI6IjIwMjYtMDUtMDFUMDA6MDA6MDArMDA6MDAifX0." + } + }) + .to_string(), + ) + .expect("auth"); + let source = SourceLocation::local_adapter( + CODEX_PROVIDER, + "test", + "0", + dir.path(), + LocationOrigin::Configured, + ); + + let evidence = CodexAdapter + .collect_account_evidence(&source, &[]) + .expect("account evidence"); + + let identity = evidence + .identity_observations + .iter() + .find(|item| item.evidence_kind == AccountEvidenceKind::AuthSnapshot) + .expect("auth snapshot identity"); + assert_eq!( + identity.observed_at.to_rfc3339(), + "2026-06-10T00:00:00+00:00" + ); + let plan = evidence.plan_observations.first().expect("plan evidence"); + assert_eq!(plan.observed_at.to_rfc3339(), "2026-05-01T00:00:00+00:00"); + } + #[test] fn codex_auth_json_reads_nested_tokens_id_token_shape() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/crates/statsai/src/main.rs b/crates/statsai/src/main.rs index 018fac5..6354031 100644 --- a/crates/statsai/src/main.rs +++ b/crates/statsai/src/main.rs @@ -5139,12 +5139,6 @@ fn split_http_rollup_sync_batch_after_budget_error(batch: &SyncBatch) -> Vec 1 { - return split_http_rollup_metadata_chunks( - batch, - batch.quota_cycle_contributions.len().div_ceil(2), - ); - } vec![batch.clone()] } @@ -5166,6 +5160,13 @@ fn has_non_quota_cycle_payload(batch: &SyncBatch) -> bool { || !batch.code_change_metrics.is_empty() } +/// Records the backend writes one statement per row for. +/// +/// Quota cycles are deliberately absent. They travel on their own splitting +/// path and the backend upserts the whole collection in a single statement, so +/// counting them here both charged them twice against the budget and made +/// `has_non_quota_cycle_payload` true for a batch that holds nothing else — +/// which returned the same quota chunk from every split and retried it forever. fn http_rollup_metadata_count(batch: &SyncBatch) -> usize { batch.sources.len() + batch.accounts.len() @@ -5173,7 +5174,6 @@ fn http_rollup_metadata_count(batch: &SyncBatch) -> usize { + batch.subscriptions.len() + batch.account_plan_observations.len() + batch.account_evidence_summaries.len() - + batch.quota_cycle_contributions.len() } fn split_http_rollup_task_chunks( @@ -5319,8 +5319,7 @@ fn estimate_http_rollup_d1_queries(batch: &SyncBatch) -> usize { + batch.source_account_assignments.len() + batch.subscriptions.len() + batch.account_plan_observations.len() - + batch.account_evidence_summaries.len() - + batch.quota_cycle_contributions.len(); + + batch.account_evidence_summaries.len(); let project_write_queries = http_rollup_project_count(batch) + http_rollup_project_location_count(batch); let daily_rollup_write_queries = http_rollup_query_chunks( @@ -5664,11 +5663,6 @@ fn split_http_rollup_metadata_chunks(batch: &SyncBatch, chunk_size: usize) -> Ve "account_evidence", chunk_size, )); - chunks.extend(split_http_rollup_single_metadata_kind( - batch, - "quota_cycles", - chunk_size, - )); chunks } @@ -5743,17 +5737,6 @@ fn split_http_rollup_single_metadata_kind( chunk }) .collect(), - "quota_cycles" => batch - .quota_cycle_contributions - .chunks(chunk_size) - .enumerate() - .map(|(index, records)| { - let mut chunk = - empty_http_rollup_chunk(batch, &format!("quota_cycles_{}", index + 1)); - chunk.quota_cycle_contributions = records.to_vec(); - chunk - }) - .collect(), _ => Vec::new(), } } @@ -15948,6 +15931,116 @@ mod tests { assert!(chunks.iter().any(|chunk| !chunk.accounts.is_empty())); } + fn test_quota_cycle_contributions( + now: DateTime, + count: usize, + ) -> Vec { + (0..count) + .map(|index| { + let reset = now + chrono::Duration::days(7 * index as i64); + statsai_core::QuotaCycleContributionV1 { + schema_version: statsai_core::QUOTA_CYCLE_CONTRIBUTION_SCHEMA_VERSION + .to_string(), + contribution_id: format!("quota_cycle_{index:032}"), + provider: "codex".to_string(), + provider_account_id: ProviderAccountId("acct".to_string()), + limit_id: Some("weekly".to_string()), + window_minutes: 10_080, + representative_reset: reset, + representative_reset_epoch_seconds: reset.timestamp(), + has_schedule_overlap: false, + daily_envelopes: Vec::new(), + boundary_slices: Vec::new(), + } + }) + .collect() + } + + fn test_quota_only_sync_batch(now: DateTime, count: usize) -> SyncBatch { + SyncBatch { + schema_version: SYNC_BATCH_SCHEMA_VERSION.to_string(), + batch_id: "batch_quota_only".to_string(), + device_id: "device".to_string(), + sources: vec![], + accounts: vec![], + source_account_assignments: vec![], + subscriptions: vec![], + account_plan_observations: vec![], + account_evidence_summaries: vec![], + events: vec![], + summaries: vec![], + task_buckets: vec![], + task_verifications: vec![], + code_change_metrics: vec![], + quota_cycle_contributions: test_quota_cycle_contributions(now, count), + authoritative_snapshot: None, + created_at: now, + } + } + + #[test] + fn a_quota_only_batch_splits_into_strictly_smaller_chunks() { + // Quota cycles carry nothing else, so the split has to make progress on + // the quota collection itself. Counting them as metadata made + // `has_non_quota_cycle_payload` true for this batch, so the splitter + // peeled the quota off "the rest" and handed back the identical chunk + // beside an empty one — which `should_retry_http_rollup_chunk_after_error` + // then retried and split the same way, forever. + let now = Utc + .with_ymd_and_hms(2026, 5, 29, 10, 12, 43) + .single() + .expect("date"); + let batch = test_quota_only_sync_batch(now, 4); + + let chunks = split_http_rollup_sync_batch_after_budget_error(&batch); + + assert!(chunks + .iter() + .all(|chunk| chunk.quota_cycle_contributions.len() + < batch.quota_cycle_contributions.len())); + assert!(chunks + .iter() + .all(|chunk| !chunk.quota_cycle_contributions.is_empty())); + assert_eq!( + chunks + .iter() + .map(|chunk| chunk.quota_cycle_contributions.len()) + .sum::(), + batch.quota_cycle_contributions.len() + ); + } + + #[test] + fn splitting_sends_each_quota_cycle_exactly_once() { + // Enough cycles to cross the metadata-per-batch limit once they were + // wrongly counted as metadata. Past that point the metadata splitter + // and the dedicated quota splitter both ran over the same batch, so + // every contribution went out twice. + let now = Utc + .with_ymd_and_hms(2026, 5, 29, 10, 12, 43) + .single() + .expect("date"); + let mut batch = + test_quota_only_sync_batch(now, HTTP_ROLLUP_METADATA_RECORDS_PER_BATCH + 10); + batch.sources = vec![SourceLocation::local_adapter( + "codex", + "test", + "0", + Path::new("/tmp/codex-quota-once"), + LocationOrigin::Configured, + )]; + + let sent = split_http_rollup_sync_batches(&batch) + .iter() + .flat_map(|chunk| chunk.quota_cycle_contributions.clone()) + .map(|contribution| contribution.contribution_id) + .collect::>(); + + let unique = sent.iter().collect::>(); + assert_eq!(sent.len(), unique.len(), "sent: {sent:?}"); + assert_eq!(unique.len(), batch.quota_cycle_contributions.len()); + } + #[test] fn http_rollup_chunk_is_resent_after_a_transient_endpoint_failure() { let now = Utc From 2d451d56aa5388042d28b9f57a5740dd3aa1a419 Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:46:15 +0300 Subject: [PATCH 05/13] Let a quota reading say the plan is current The provider names the plan while serving a request, so a quota reading is evidence of the plan as of that moment -- and a fresher one than `auth.json`, which can sit unchanged on disk for weeks. It was emitted with `is_current_snapshot: false`, so an account whose logs report "plus" every day still read as `last_detected` the moment its last declared provider period ran out, and it dropped off every current-plan surface. Still no invented billing window: the reading declares no period, so `active_from` and `active_until` stay empty and the timeline keeps showing it as an observation rather than a provider period. --- crates/statsai-store/src/account_plan.rs | 10 ++++++- crates/statsai-store/src/quota.rs | 36 ++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/crates/statsai-store/src/account_plan.rs b/crates/statsai-store/src/account_plan.rs index 238b50f..695be56 100644 --- a/crates/statsai-store/src/account_plan.rs +++ b/crates/statsai-store/src/account_plan.rs @@ -717,9 +717,17 @@ impl Store { raw_plan_name: run.raw_plan_name.to_string(), plan_name: normalize_plan_name(run.raw_plan_name), observed_at: quota.observed_at, + // No period: the provider reported a plan while serving a + // request, it did not declare a billing window. It did + // report it *as of that moment*, though, which is what + // `is_current_snapshot` means -- and it is fresher than + // `auth.json`, which can sit on disk unchanged for weeks. + // Leaving this false meant an account whose logs say "plus" + // every day still read as `last_detected` as soon as its + // last declared provider period ran out. active_from: None, active_until: None, - is_current_snapshot: false, + is_current_snapshot: true, evidence_kind: AccountEvidenceKind::QuotaStatus, confidence: Confidence::High, parser_version: "quota-plan-evidence.v1".to_string(), diff --git a/crates/statsai-store/src/quota.rs b/crates/statsai-store/src/quota.rs index e9f9442..01b39d0 100644 --- a/crates/statsai-store/src/quota.rs +++ b/crates/statsai-store/src/quota.rs @@ -1685,6 +1685,42 @@ mod tests { (source.source_id, account_id) } + #[test] + fn quota_plan_evidence_speaks_for_the_moment_it_was_read() { + // The provider names the plan while serving a request, so the reading + // is evidence of the plan *now* even though it declares no billing + // window. Without that, an account whose logs say "plus" every day + // still read as `last_detected` once its last declared provider period + // ran out, and it dropped off every current-plan surface. + let store = Store::in_memory().expect("store"); + let observed_at = DateTime::from_timestamp(1_787_227_200, 0).expect("observed at"); + let reset_at = observed_at + Duration::days(7); + let (source_id, _) = assigned_source(&store, observed_at - Duration::days(1)); + let record = sample_record( + source_id.clone(), + "quota-current", + "quota-current-v1", + observed_at, + reset_at.timestamp(), + "primary", + 10_080, + 25.0, + ); + store + .upsert_quota_observations(std::slice::from_ref(&record)) + .expect("quota row"); + store + .rebuild_quota_plan_observations_for_source(&source_id) + .expect("plan rebuild"); + + let plans = store.account_plan_observations().expect("plan evidence"); + assert_eq!(plans.len(), 1); + assert!(plans[0].is_current_snapshot); + // Still no invented billing period. + assert_eq!(plans[0].active_from, None); + assert_eq!(plans[0].active_until, None); + } + #[test] fn quota_plan_evidence_rebuild_replaces_corrected_plan_rows() { let store = Store::in_memory().expect("store"); From 6b3ebd93987837a4193e41b2700d58aafa1219d4 Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Tue, 25 Aug 2026 22:11:55 +0300 Subject: [PATCH 06/13] Keep account email and provider id on the sync payload Stripping them left every account in the dashboard showing as a bare `acct_` hash with no way to tell one of your own accounts from another, which is most of the reason accounts sync at all. The account's own identity is not what this feature had to make private: the new evidence types are, and those carry hashes under their own contracts. Server-side canonicalisation already prefers the hashes and only falls back to these, so it was unaffected either way. `plan_name` still goes, because a plan is evidence now rather than an account attribute. --- crates/statsai/src/main.rs | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/crates/statsai/src/main.rs b/crates/statsai/src/main.rs index 6354031..30056b3 100644 --- a/crates/statsai/src/main.rs +++ b/crates/statsai/src/main.rs @@ -9821,8 +9821,13 @@ fn sanitize_account_for_sync(mut account: ProviderAccount) -> ProviderAccount { if !matches!(account.identity_source, IdentitySource::UserConfigured) { account.account_label = None; } - account.provider_user_id = None; - account.email = None; + // The email and provider user id stay. They are how a person tells one of + // their own accounts from another in the dashboard, which is the whole + // point of syncing accounts at all; stripping them left every account + // showing as a bare `acct_` hash. The new evidence types are the ones that + // must never carry them -- those travel as hashes and are covered by their + // own contracts. `plan_name` still goes, because a plan is now evidence + // rather than an account attribute. account.plan_name = None; account } @@ -20688,8 +20693,14 @@ mod tests { let sanitized = sanitize_account_for_sync(account); assert_eq!(sanitized.account_label.as_deref(), Some("personal")); - assert_eq!(sanitized.provider_user_id, None); - assert_eq!(sanitized.email, None); + // The account's own identity travels: without it the dashboard can + // only name an account by its `acct_` hash, and telling your own + // accounts apart is why they sync in the first place. + assert_eq!( + sanitized.provider_user_id.as_deref(), + Some("provider-user-secret") + ); + assert_eq!(sanitized.email.as_deref(), Some("private@example.com")); assert_eq!( sanitized.provider_user_id_hash.as_deref(), Some("a".repeat(64).as_str()) @@ -20698,11 +20709,8 @@ mod tests { sanitized.email_hash.as_deref(), Some("b".repeat(64).as_str()) ); + // A plan is evidence now, not an account attribute. assert_eq!(sanitized.plan_name, None); - - let payload = serde_json::to_string(&sanitized).expect("serialize sanitized account"); - assert!(!payload.contains("provider-user-secret")); - assert!(!payload.contains("private@example.com")); } #[test] From e41b536aea013b1ee403345d09174427fcfc2caa Mon Sep 17 00:00:00 2001 From: Dmitry Starkov <21260939+starkdmi@users.noreply.github.com> Date: Wed, 26 Aug 2026 01:08:58 +0300 Subject: [PATCH 07/13] Read the Codex identity evidence this generation actually writes The account attribution shipped inert: every gate in the collector was tuned to artifacts the current Codex build no longer produces, so the evidence tables stayed empty and every source kept the one-account-per- directory attribution this feature was meant to replace. - Telemetry identity was only accepted from `codex.conversation_starts` events, which this generation emits a handful of times in total. The account id rides on ordinary events (`codex.turn_ttft`, `codex.tool_decision`, ...), so any single declared `codex.*` event in the trusted structured prefix now qualifies. The free-text cut, the sole-attribute rule, and the double-event rejection are unchanged. - Auth reloads moved to the `codex_login::auth::manager` target, and the message now closes a tracing span (`app_server.request{...}: Reloading auth for account `). The line is accepted at start of body or right after a span close, nothing may follow the account id, and a match at or past the first free-text field stays inert. - Multi-account setups keep swapped-out logins as `auth-