From d520c3b07a4b74326f09d499eaba49775f0c0697 Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Wed, 26 Aug 2026 12:25:40 +1000 Subject: [PATCH] feat(projects): add safe storage shadow migration Signed-off-by: Matt Toohey --- docs/project-storage-migration.md | 65 + .../20260826010000_create_project_storage.sql | 93 + src-tauri/src/commands/layout.rs | 282 +- src-tauri/src/commands/mod.rs | 1 + src-tauri/src/commands/projects.rs | 2290 +++++++++++++++++ src-tauri/src/lib.rs | 16 +- src-tauri/src/persistence.rs | 382 +++ src-tauri/src/services/e2e_mode.rs | 4 + src-tauri/src/services/goose_config.rs | 49 +- .../api/projects.shadow-import.test.ts | 55 + src/features/projects/api/projects.ts | 37 + 11 files changed, 2987 insertions(+), 287 deletions(-) create mode 100644 docs/project-storage-migration.md create mode 100644 src-tauri/migrations/20260826010000_create_project_storage.sql create mode 100644 src-tauri/src/commands/projects.rs create mode 100644 src-tauri/src/persistence.rs create mode 100644 src/features/projects/api/projects.shadow-import.test.ts diff --git a/docs/project-storage-migration.md b/docs/project-storage-migration.md new file mode 100644 index 000000000..3a75cf98f --- /dev/null +++ b/docs/project-storage-migration.md @@ -0,0 +1,65 @@ +# Project storage migration + +Projects and their session associations are moving from Goose-owned Markdown and session metadata to Berd's existing `berd.sqlite` database. The migration is deliberately monotonic and does not activate the new authority in this change. + +## Ownership boundary + +| Data | Legacy authority | Native authority | +| --- | --- | --- | +| Project records, order, archive state, and workspaces | Goose project Markdown | Berd SQLite | +| Session-to-project association | Goose session `_meta.projectId` | Berd `session_projects` | +| Conversation and session lifecycle | Goose | Goose | +| Project prompt composition | Goose | Berd immediately before dispatch | + +The last three ownership changes must activate together. Switching only project CRUD would leave Goose reading retained Markdown while Berd writes SQLite, and switching only prompt composition would duplicate or omit model instructions. + +## Persisted phases + +- `legacy`: Goose remains the only authority. +- `shadow`: Berd mirrors the fixed Goose project directory, but all product reads and writes still use Goose. +- `native`: Berd owns project reads, writes, associations, and prompt lookup. The database trigger prevents a transition back to `legacy` or `shadow`. + +This change exposes no native-activation command. It starts a retryable shadow import when project listing first runs and leaves the existing Goose request path unchanged. + +## Shadow import contract + +The importer resolves the same process/login-shell `GOOSE_PATH_ROOT` precedence used to launch `goose serve`, then scans only `/projects`. + +- Inputs are deterministically ordered and limited to 1,000 regular, non-symlink Markdown files of at most 1 MiB each. +- Exact source bytes are SHA-256 fingerprinted and copied to a content-addressed Berd recovery directory before the SQLite transaction. +- The recovery directory is anchored below Berd app data, rejects symlink components, and has a 1 GiB aggregate safety limit. Reaching the limit fails the passive scan without changing Goose-owned product behavior. +- Project rows, workspaces, receipts, migration status, and the phase change commit in one transaction. +- Re-running is idempotent. Changes continue to refresh the mirror while Goose is authoritative. +- Goose mutations force a reconciliation; read-triggered reconciliations are limited to once per 30 seconds to avoid repeatedly hashing the full collection. +- A database-backed scan generation rejects stale results and stale failures across windows or concurrent Berd processes. Unchanged rescans do not manufacture collection revisions. +- A missing or changed source root cannot erase an existing mirror. +- Unknown frontmatter properties are preserved as JSON for compatibility. +- Original Goose files are never changed or deleted. +- A Berd build refuses to read or rewrite a migration run created by a newer schema version. + +Database corruption now writes a durable recovery marker, preserves the database/WAL/SHM files, and blocks subsequent startup. Berd no longer silently creates an empty authoritative database after corruption. + +## Session loading in the native end state + +Goose will continue to load session conversations. Berd overlays the association using `(session_backend_id, session_id)`, where the backend namespace is stable across provider or harness changes. A stored nullable row is an explicit "no project" tombstone and wins over legacy metadata; absence of a row permits one-time legacy backfill. + +The project must be resolved from SQLite immediately before every foreground or queued dispatch. Persistence failure must retain the queued message and prevent the user turn from being committed. Renderer local storage remains a presentation cache only. + +## Native activation blockers + +Native activation is not safe until all of these are resolved in one pinned Berd/Goose release: + +1. Goose commit `063694cf769269c1f151416605687991fdcbc496`, currently pinned in `goose-backend.lock.json`, is not reachable from the configured upstream. That exact source must be recovered and verified before its two project-prompt injection paths can be removed or capability-gated. +2. Session create and fork need a durable operation correlation so a crash between Goose session allocation and Berd association persistence can reconcile on restart. +3. Every session list/info/import/fork/provider-recovery path must overlay Berd associations, including archived projects and paginated results. +4. Project changes must be resolved at dispatch time; queued sends must fail closed on database errors, and project moves during an active turn need a defined next-turn policy. +5. Cross-window mutations need revision-aware broadcasts/refetch, and product callers must await association persistence before updating UI state. +6. Downgrade after native writes is not lossless because retained Markdown becomes stale. A rollback build must retain the Berd reader or run a tested reverse export; otherwise downgrade must be explicitly unsupported. +7. External ACP providers have no generally revocable system-prompt channel. Clearing or moving a project cannot remove project context already sent in-band, which requires a documented product decision. +8. Activation must prove the latest claimed scan completed successfully against the current source root and that no newer scan is in flight. A bounded-backup maintenance and user-visible failure policy is also needed before the passive 1 GiB limit can become an authoritative migration dependency. +9. Filesystem checks prevent direct symlink inputs and redirected backup roots, but portable Rust path APIs cannot make the whole directory walk atomic against a privileged process swapping parent directories. Native activation needs an anchored directory-handle implementation or an explicit threat-model decision. +10. Native project deletion must tombstone affected session associations and increment their revisions in the same transaction before deleting the project. The schema deliberately restricts implicit foreign-key deletion so this cannot be skipped. +11. Compatibility needs shared golden fixtures that run the same legacy sources through Goose, renderer normalization, and Berd import. The Rust compatibility cases cover known workspace and filename variants but do not prove the cross-language contract. +12. Worst-case shadow scan and SQLite lock time needs measurement at the 1,000-file/1 GiB input bound. Durable failure status exists in SQLite, but user-visible reporting or telemetry is still required before activation. + +Until these gates pass, `shadow` is the only safe deployed state. diff --git a/src-tauri/migrations/20260826010000_create_project_storage.sql b/src-tauri/migrations/20260826010000_create_project_storage.sql new file mode 100644 index 000000000..be010103d --- /dev/null +++ b/src-tauri/migrations/20260826010000_create_project_storage.sql @@ -0,0 +1,93 @@ +CREATE TABLE project_storage_state ( + singleton_id INTEGER PRIMARY KEY CHECK (singleton_id = 1), + phase TEXT NOT NULL CHECK (phase IN ('legacy', 'shadow', 'native')), + collection_revision INTEGER NOT NULL DEFAULT 0 CHECK (collection_revision >= 0), + shadow_scan_generation INTEGER NOT NULL DEFAULT 0 CHECK (shadow_scan_generation >= 0), + shadow_completed_at TEXT, + native_activated_at TEXT +); + +INSERT INTO project_storage_state ( + singleton_id, + phase, + collection_revision +) VALUES (1, 'legacy', 0); + +CREATE TRIGGER project_storage_phase_is_monotonic +BEFORE UPDATE OF phase ON project_storage_state +WHEN (OLD.phase = 'legacy' AND NEW.phase = 'native') + OR (OLD.phase = 'shadow' AND NEW.phase = 'legacy') + OR (OLD.phase = 'native' AND NEW.phase != 'native') +BEGIN + SELECT RAISE(ABORT, 'project storage phase cannot move backwards'); +END; + +CREATE TABLE projects ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT NOT NULL, + prompt TEXT NOT NULL, + icon TEXT NOT NULL DEFAULT '', + color TEXT NOT NULL DEFAULT '', + use_worktrees INTEGER NOT NULL DEFAULT 0 CHECK (use_worktrees IN (0, 1)), + order_index INTEGER NOT NULL DEFAULT 0, + archived_at TEXT, + artifact_json TEXT, + chat_groups_json TEXT, + extra_properties_json TEXT NOT NULL DEFAULT '{}', + revision INTEGER NOT NULL DEFAULT 1 CHECK (revision >= 1), + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE INDEX projects_active_order_idx +ON projects (archived_at, order_index, id); + +CREATE TABLE project_workspaces ( + project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + position INTEGER NOT NULL CHECK (position >= 0), + workspace_id TEXT NOT NULL, + path TEXT NOT NULL, + kind TEXT NOT NULL, + source TEXT NOT NULL, + branch TEXT, + repository_path TEXT, + worktree_path TEXT, + startup_mode TEXT NOT NULL, + PRIMARY KEY (project_id, position) +); + +CREATE TABLE session_projects ( + session_backend_id TEXT NOT NULL, + session_id TEXT NOT NULL, + project_id TEXT REFERENCES projects(id) ON DELETE RESTRICT, + source TEXT NOT NULL CHECK (source IN ('legacy-backfill', 'native')), + revision INTEGER NOT NULL DEFAULT 1 CHECK (revision >= 1), + updated_at TEXT NOT NULL, + PRIMARY KEY (session_backend_id, session_id) +); + +CREATE INDEX session_projects_project_idx +ON session_projects (project_id); + +CREATE TABLE project_migration_runs ( + migration_key TEXT PRIMARY KEY, + version INTEGER NOT NULL CHECK (version >= 1), + scan_generation INTEGER NOT NULL DEFAULT 0 CHECK (scan_generation >= 0), + state TEXT NOT NULL CHECK (state IN ('running', 'complete', 'failed')), + checkpoint_json TEXT NOT NULL DEFAULT '{}', + completed_at TEXT, + last_error TEXT, + updated_at TEXT NOT NULL +); + +CREATE TABLE legacy_project_imports ( + source_path TEXT PRIMARY KEY, + source_fingerprint TEXT NOT NULL, + project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE RESTRICT, + backup_path TEXT NOT NULL, + imported_at TEXT NOT NULL +); + +CREATE UNIQUE INDEX legacy_project_imports_project_idx +ON legacy_project_imports (project_id); diff --git a/src-tauri/src/commands/layout.rs b/src-tauri/src/commands/layout.rs index 5b8dceab0..a7660b272 100644 --- a/src-tauri/src/commands/layout.rs +++ b/src-tauri/src/commands/layout.rs @@ -1,16 +1,12 @@ -use chrono::Utc; +use crate::persistence::BerdPersistenceState; +#[cfg(test)] +use crate::persistence::DATABASE_FILENAME; use serde::{Deserialize, Serialize, Serializer}; use sqlx::pool::PoolConnection; -use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions}; use sqlx::{FromRow, Row, Sqlite, SqlitePool}; use std::collections::{HashMap, HashSet}; -use std::fs::{self, File}; -use std::io::Read; -use std::path::{Path, PathBuf}; -use std::time::Duration; use uuid::Uuid; -const DATABASE_FILENAME: &str = "berd.sqlite"; const MIN_CENTER: f64 = -1_000_000.0; const MAX_CENTER: f64 = 1_000_000.0; const MIN_SIZE: f64 = 1.0; @@ -22,57 +18,7 @@ const MAX_TITLE_OVERRIDE_LENGTH: usize = 200; const MAX_ITEMS: usize = 500; pub const HOME_LAYOUT_ID: &str = "home"; -#[derive(Clone, Debug)] -pub struct LayoutState { - pool: SqlitePool, -} - -impl LayoutState { - pub async fn new(app_data_dir: PathBuf) -> Result { - fs::create_dir_all(&app_data_dir) - .map_err(|error| format!("Failed to create app data directory: {error}"))?; - - Self::new_for_path(app_data_dir.join(DATABASE_FILENAME)).await - } - - #[cfg(test)] - async fn new_for_tests(db_path: PathBuf) -> Result { - Self::new_for_path(db_path).await - } - - async fn new_for_path(db_path: PathBuf) -> Result { - if database_file_has_invalid_header(&db_path)? { - rename_corrupt_database_files(&db_path)?; - } - match open_migrated_pool(&db_path).await { - Ok(pool) => Ok(Self { pool }), - Err(error) if is_sqlite_corruption_error(&error) => { - rename_corrupt_database_files(&db_path)?; - let pool = open_migrated_pool(&db_path).await.map_err(|retry_error| { - format!("Failed to recreate layout database: {retry_error}") - })?; - Ok(Self { pool }) - } - Err(error) => Err(format!("Failed to initialize layout database: {error}")), - } - } -} - -fn database_file_has_invalid_header(db_path: &Path) -> Result { - const SQLITE_HEADER: &[u8] = b"SQLite format 3\0"; - - if !db_path.exists() { - return Ok(false); - } - - let mut file = File::open(db_path) - .map_err(|error| format!("Failed to inspect layout database header: {error}"))?; - let mut header = [0; SQLITE_HEADER.len()]; - let bytes_read = file - .read(&mut header) - .map_err(|error| format!("Failed to inspect layout database header: {error}"))?; - Ok(bytes_read != 0 && (bytes_read < SQLITE_HEADER.len() || header.as_slice() != SQLITE_HEADER)) -} +pub type LayoutState = BerdPersistenceState; #[derive(Serialize, Clone, Debug, PartialEq)] #[serde(rename_all = "camelCase")] @@ -301,102 +247,6 @@ pub async fn reset_layout( reset_layout_in_pool(&state.pool, request).await } -async fn open_migrated_pool(db_path: &Path) -> Result { - let options = SqliteConnectOptions::new() - .filename(db_path) - .create_if_missing(true) - .journal_mode(SqliteJournalMode::Wal) - .busy_timeout(Duration::from_millis(5_000)); - let pool = SqlitePoolOptions::new() - .max_connections(5) - .connect_with(options) - .await?; - - if let Err(error) = sqlx::migrate!("./migrations").run(&pool).await { - pool.close().await; - return Err(error.into()); - } - if let Err(error) = read_initial_state(&pool).await { - pool.close().await; - return Err(error); - } - Ok(pool) -} - -async fn read_initial_state(pool: &SqlitePool) -> Result<(), sqlx::Error> { - sqlx::query("SELECT layout_id FROM layout_state WHERE layout_id = ?") - .bind(HOME_LAYOUT_ID) - .fetch_one(pool) - .await?; - Ok(()) -} - -fn is_sqlite_corruption_error(error: &sqlx::Error) -> bool { - let mut current: Option<&(dyn std::error::Error + 'static)> = Some(error); - while let Some(error) = current { - if let Some(database_error) = error.downcast_ref::().and_then(|error| { - if let sqlx::Error::Database(database_error) = error { - Some(database_error) - } else { - None - } - }) { - if is_sqlite_corruption_code(database_error.code().as_deref()) { - return true; - } - } - current = error.source(); - } - false -} - -fn is_sqlite_corruption_code(code: Option<&str>) -> bool { - code.and_then(|code| code.parse::().ok()) - .is_some_and(|code| matches!(code & 0xff, 11 | 26)) -} - -fn rename_corrupt_database_files(db_path: &Path) -> Result<(), String> { - let suffix = format!(".corrupt-{}", Utc::now().format("%Y%m%dT%H%M%SZ")); - let mut renamed = Vec::new(); - let mut errors = Vec::new(); - - for path in [ - db_path.to_path_buf(), - PathBuf::from(format!("{}-wal", db_path.display())), - PathBuf::from(format!("{}-shm", db_path.display())), - ] { - if path.exists() { - let file_name = path - .file_name() - .ok_or_else(|| format!("Invalid database path: {}", path.display()))? - .to_string_lossy(); - let renamed_path = path.with_file_name(format!("{file_name}{suffix}")); - match fs::rename(&path, &renamed_path) { - Ok(()) => renamed.push((path, renamed_path)), - Err(error) => errors.push(format!( - "Failed to preserve corrupt database file {} as {}: {error}", - path.display(), - renamed_path.display() - )), - } - } - } - - for (path, renamed_path) in &renamed { - log::warn!( - "Preserved corrupt layout database file {} as {}", - path.display(), - renamed_path.display() - ); - } - - if errors.is_empty() { - Ok(()) - } else { - Err(errors.join("; ")) - } -} - fn require_supported_layout_id(layout_id: String) -> Result { let layout_id = layout_id.trim().to_string(); if layout_id.is_empty() { @@ -912,50 +762,10 @@ fn db_error(error: sqlx::Error) -> String { mod tests { use super::*; use serde_json::json; - use sqlx::error::{DatabaseError, ErrorKind}; - use std::borrow::Cow; - use std::fmt; + use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions}; + use std::time::Duration; use tempfile::{tempdir, TempDir}; - #[derive(Debug)] - struct FakeDatabaseError { - code: &'static str, - } - - impl fmt::Display for FakeDatabaseError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(formatter, "fake sqlite error {}", self.code) - } - } - - impl std::error::Error for FakeDatabaseError {} - - impl DatabaseError for FakeDatabaseError { - fn message(&self) -> &str { - "fake sqlite error" - } - - fn code(&self) -> Option> { - Some(Cow::Borrowed(self.code)) - } - - fn as_error(&self) -> &(dyn std::error::Error + Send + Sync + 'static) { - self - } - - fn as_error_mut(&mut self) -> &mut (dyn std::error::Error + Send + Sync + 'static) { - self - } - - fn into_error(self: Box) -> Box { - self - } - - fn kind(&self) -> ErrorKind { - ErrorKind::Other - } - } - struct TestState { state: LayoutState, _dir: TempDir, @@ -1146,86 +956,6 @@ mod tests { assert_eq!(layout.items[0].target_id, "session-1"); } - #[tokio::test] - async fn corrupt_database_files_are_renamed_with_matching_suffix() { - let dir = tempdir().expect("tempdir"); - let db_path = dir.path().join(DATABASE_FILENAME); - fs::write(&db_path, "not sqlite").expect("db"); - fs::write(format!("{}-wal", db_path.display()), "wal").expect("wal"); - fs::write(format!("{}-shm", db_path.display()), "shm").expect("shm"); - - let state = LayoutState::new_for_tests(db_path.clone()) - .await - .expect("recovered state"); - assert_eq!( - read_layout(&state.pool, HOME_LAYOUT_ID) - .await - .expect("layout") - .camera - .zoom_bps, - DEFAULT_ZOOM_BPS - ); - - let renamed: Vec = fs::read_dir(dir.path()) - .expect("read dir") - .filter_map(Result::ok) - .map(|entry| entry.file_name().to_string_lossy().into_owned()) - .filter(|name| name.contains(".corrupt-")) - .collect(); - assert_eq!(renamed.len(), 3); - let suffixes: HashSet = renamed - .iter() - .map(|name| name.split(".corrupt-").nth(1).unwrap().to_string()) - .collect(); - assert_eq!(suffixes.len(), 1); - } - - #[tokio::test] - async fn non_corruption_open_errors_do_not_rename_database_path() { - let dir = tempdir().expect("tempdir"); - let db_path = dir.path().join(DATABASE_FILENAME); - fs::create_dir(&db_path).expect("db path directory"); - - let error = LayoutState::new_for_tests(db_path.clone()) - .await - .expect_err("directory path should not be recoverable corruption"); - - assert!(error.contains("Failed to inspect layout database header")); - assert!(db_path.is_dir()); - let renamed: Vec = fs::read_dir(dir.path()) - .expect("read dir") - .filter_map(Result::ok) - .map(|entry| entry.file_name().to_string_lossy().into_owned()) - .filter(|name| name.contains(".corrupt-")) - .collect(); - assert!(renamed.is_empty()); - } - - #[test] - fn corruption_detector_matches_migration_errors_and_extended_codes() { - let top_level_corruption = sqlx::Error::database(FakeDatabaseError { code: "11" }); - assert!(is_sqlite_corruption_error(&top_level_corruption)); - - let extended_corruption = sqlx::Error::database(FakeDatabaseError { code: "267" }); - assert!(is_sqlite_corruption_error(&extended_corruption)); - - let migration_corruption = - sqlx::Error::Migrate(Box::new(sqlx::migrate::MigrateError::ExecuteMigration( - sqlx::Error::database(FakeDatabaseError { code: "267" }), - 20260519180000, - ))); - assert!(is_sqlite_corruption_error(&migration_corruption)); - - let migration_not_a_database = - sqlx::Error::Migrate(Box::new(sqlx::migrate::MigrateError::Execute( - sqlx::Error::database(FakeDatabaseError { code: "26" }), - ))); - assert!(is_sqlite_corruption_error(&migration_not_a_database)); - - let constraint_error = sqlx::Error::database(FakeDatabaseError { code: "2067" }); - assert!(!is_sqlite_corruption_error(&constraint_error)); - } - #[tokio::test] async fn save_layout_items_replaces_rows_preserves_existing_sort_and_bumps_item_revision_only() { diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 97b28f72b..1f950ff0b 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -43,6 +43,7 @@ pub mod path_resolver; mod pocket_audio_player; pub mod pocket_voice; pub mod project_icons; +pub mod projects; pub mod pull_requests; pub mod renderer; pub mod runtime_config; diff --git a/src-tauri/src/commands/projects.rs b/src-tauri/src/commands/projects.rs new file mode 100644 index 000000000..85a463f41 --- /dev/null +++ b/src-tauri/src/commands/projects.rs @@ -0,0 +1,2290 @@ +use crate::persistence::BerdPersistenceState; +use crate::services::{dir_env, e2e_mode::E2eMode, env_key, goose_config}; +use chrono::Utc; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use sqlx::{Row, Sqlite, Transaction}; +use std::collections::{BTreeMap, HashSet}; +use std::ffi::OsString; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use tauri::{AppHandle, Manager, State}; +use tokio::sync::Mutex; + +const MIGRATION_KEY: &str = "legacy-goose-projects"; +const MIGRATION_VERSION: i64 = 1; +const MAX_PROJECT_FILES: usize = 1_000; +const MAX_PROJECT_FILE_BYTES: u64 = 1024 * 1024; +const MAX_BACKUP_BYTES: u64 = MAX_PROJECT_FILES as u64 * MAX_PROJECT_FILE_BYTES; +static SHADOW_IMPORT_LOCK: Mutex<()> = Mutex::const_new(()); + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ProjectStorageStatus { + phase: String, + collection_revision: i64, + claimed_scan_generation: i64, + shadow_completed_at: Option, + native_activated_at: Option, + migration_version: Option, + migration_scan_generation: Option, + migration_state: Option, + migration_error: Option, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ShadowImportReport { + phase: String, + scan_generation: i64, + source_count: usize, + imported: usize, + updated: usize, + removed: usize, + unchanged: usize, + superseded: bool, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacySessionProjectBackfillRequest { + session_backend_id: String, + session_id: String, + project_id: String, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct UpdateSessionProjectRequest { + session_backend_id: String, + session_id: String, + project_id: Option, + expected_revision: i64, +} + +#[derive(Debug, Serialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct SessionProjectAssociation { + session_backend_id: String, + session_id: String, + has_association: bool, + project_id: Option, + revision: Option, +} + +#[derive(Debug)] +struct LegacyProject { + source_path: String, + fingerprint: String, + backup_path: String, + project: StoredProject, +} + +#[derive(Debug)] +struct LegacyScan { + source_present: bool, + source_root: Option, + projects: Vec, +} + +#[derive(Debug)] +struct StoredProject { + id: String, + name: String, + description: String, + prompt: String, + icon: String, + color: String, + use_worktrees: bool, + order: i64, + archived_at: Option, + artifact_json: Option, + chat_groups_json: Option, + properties_json: String, + workspaces: Vec, +} + +#[derive(Debug)] +struct StoredWorkspace { + id: String, + path: String, + kind: String, + source: String, + branch: Option, + repository_path: Option, + worktree_path: Option, + startup_mode: String, +} + +#[derive(Debug, Default, Deserialize)] +struct LegacyFrontmatter { + #[serde(default)] + name: String, + #[serde(default)] + description: String, + #[serde(default, flatten)] + properties: BTreeMap, +} + +#[tauri::command] +pub async fn get_project_storage_status( + state: State<'_, BerdPersistenceState>, +) -> Result { + project_storage_status(state.pool()).await +} + +#[tauri::command] +pub async fn shadow_import_legacy_projects( + app: AppHandle, + state: State<'_, BerdPersistenceState>, +) -> Result { + let _import_guard = SHADOW_IMPORT_LOCK.lock().await; + let scan_generation = claim_shadow_scan(state.pool()).await?; + let app_data_dir = app + .path() + .app_data_dir() + .map_err(|error| format!("Failed to resolve Berd app data directory: {error}"))?; + let goose_data_dir = if let Some(e2e_mode) = app.try_state::() { + e2e_mode.goose_data_dir() + } else { + let shell_env = dir_env::capture_home_interactive_env().await; + goose_config::data_dir_from_values( + std::env::var_os(goose_config::GOOSE_PATH_ROOT_ENV), + env_key::get(&shell_env, goose_config::GOOSE_PATH_ROOT_ENV).map(OsString::from), + )? + }; + let source_dir = goose_data_dir.join("projects"); + let backup_dir = app_data_dir + .join("project-migration-backups") + .join(format!("v{MIGRATION_VERSION}")); + + let scan = match tauri::async_runtime::spawn_blocking(move || { + scan_legacy_projects(&source_dir, &backup_dir) + }) + .await + { + Ok(Ok(scan)) => scan, + Ok(Err(error)) => { + record_import_failure(state.pool(), scan_generation, &error).await?; + return Err(error); + } + Err(error) => { + let error = format!("Legacy project scan task failed: {error}"); + record_import_failure(state.pool(), scan_generation, &error).await?; + return Err(error); + } + }; + + match apply_shadow_import(state.pool(), scan, scan_generation).await { + Ok(report) => Ok(report), + Err(error) if error.contains("disabled after native storage activation") => Err(error), + Err(error) => { + record_import_failure(state.pool(), scan_generation, &error).await?; + Err(error) + } + } +} + +#[tauri::command] +pub async fn backfill_legacy_session_project( + state: State<'_, BerdPersistenceState>, + request: LegacySessionProjectBackfillRequest, +) -> Result { + validate_session_key(&request.session_backend_id, &request.session_id)?; + validate_stored_project_id(&request.project_id)?; + backfill_legacy_association(state.pool(), request).await +} + +#[tauri::command] +pub async fn resolve_session_project( + state: State<'_, BerdPersistenceState>, + session_backend_id: String, + session_id: String, +) -> Result { + validate_session_key(&session_backend_id, &session_id)?; + read_session_association(state.pool(), &session_backend_id, &session_id).await +} + +#[tauri::command] +pub async fn update_native_session_project( + state: State<'_, BerdPersistenceState>, + request: UpdateSessionProjectRequest, +) -> Result { + validate_session_key(&request.session_backend_id, &request.session_id)?; + if request.expected_revision < 0 { + return Err("Session project expected revision must not be negative".to_string()); + } + if let Some(project_id) = &request.project_id { + validate_stored_project_id(project_id)?; + } + update_native_association(state.pool(), request).await +} + +async fn project_storage_status(pool: &sqlx::SqlitePool) -> Result { + ensure_supported_migration_version(pool).await?; + let row = sqlx::query( + "SELECT phase, collection_revision, shadow_scan_generation, shadow_completed_at, native_activated_at \ + FROM project_storage_state WHERE singleton_id = 1", + ) + .fetch_one(pool) + .await + .map_err(db_error)?; + let migration = + sqlx::query( + "SELECT version, scan_generation, state, last_error FROM project_migration_runs WHERE migration_key = ?", + ) + .bind(MIGRATION_KEY) + .fetch_optional(pool) + .await + .map_err(db_error)?; + + Ok(ProjectStorageStatus { + phase: row.get("phase"), + collection_revision: row.get("collection_revision"), + claimed_scan_generation: row.get("shadow_scan_generation"), + shadow_completed_at: row.get("shadow_completed_at"), + native_activated_at: row.get("native_activated_at"), + migration_version: migration.as_ref().map(|row| row.get("version")), + migration_scan_generation: migration.as_ref().map(|row| row.get("scan_generation")), + migration_state: migration.as_ref().map(|row| row.get("state")), + migration_error: migration.and_then(|row| row.get("last_error")), + }) +} + +async fn claim_shadow_scan(pool: &sqlx::SqlitePool) -> Result { + let mut transaction = pool.begin_with("BEGIN IMMEDIATE").await.map_err(db_error)?; + ensure_supported_migration_version_in_transaction(&mut transaction).await?; + let phase: String = + sqlx::query_scalar("SELECT phase FROM project_storage_state WHERE singleton_id = 1") + .fetch_one(&mut *transaction) + .await + .map_err(db_error)?; + if phase == "native" { + return Err( + "Legacy project import is disabled after native storage activation".to_string(), + ); + } + let generation: i64 = sqlx::query_scalar( + "UPDATE project_storage_state SET shadow_scan_generation = shadow_scan_generation + 1 \ + WHERE singleton_id = 1 RETURNING shadow_scan_generation", + ) + .fetch_one(&mut *transaction) + .await + .map_err(db_error)?; + transaction.commit().await.map_err(db_error)?; + Ok(generation) +} + +fn scan_legacy_projects(source_dir: &Path, backup_dir: &Path) -> Result { + if !source_dir.exists() { + return Ok(LegacyScan { + source_present: false, + source_root: None, + projects: Vec::new(), + }); + } + + let unresolved_root_metadata = fs::symlink_metadata(source_dir).map_err(|error| { + format!( + "Failed to inspect legacy projects directory {}: {error}", + source_dir.display() + ) + })?; + if unresolved_root_metadata.file_type().is_symlink() { + return Err(format!( + "Legacy projects root must not be a symlink: {}", + source_dir.display() + )); + } + let source_root = source_dir.canonicalize().map_err(|error| { + format!( + "Failed to canonicalize legacy projects directory {}: {error}", + source_dir.display() + ) + })?; + let root_metadata = fs::symlink_metadata(&source_root).map_err(|error| { + format!( + "Failed to inspect legacy projects directory {}: {error}", + source_root.display() + ) + })?; + if !root_metadata.is_dir() || root_metadata.file_type().is_symlink() { + return Err(format!( + "Legacy projects root is not a regular directory: {}", + source_root.display() + )); + } + + let mut paths = Vec::new(); + for entry in fs::read_dir(&source_root).map_err(|error| { + format!( + "Failed to read legacy projects directory {}: {error}", + source_root.display() + ) + })? { + let entry = + entry.map_err(|error| format!("Failed to read legacy project entry: {error}"))?; + let path = entry.path(); + if path.extension().and_then(|extension| extension.to_str()) != Some("md") { + continue; + } + if paths.len() == MAX_PROJECT_FILES { + return Err(format!( + "Legacy project import exceeds the {MAX_PROJECT_FILES}-file safety limit" + )); + } + paths.push(path); + } + paths.sort_by(|left, right| left.file_name().cmp(&right.file_name())); + + let (backup_dir, mut backup_bytes) = prepare_backup_directory(backup_dir, &source_root)?; + + let mut candidates = Vec::with_capacity(paths.len()); + let mut ids = HashSet::new(); + for path in paths { + let (bytes, canonical_path) = read_bounded_regular_file(&source_root, &path)?; + let raw = std::str::from_utf8(&bytes).map_err(|_| { + format!( + "Legacy project is not valid UTF-8: {}", + canonical_path.display() + ) + })?; + let id = canonical_path + .file_stem() + .and_then(|stem| stem.to_str()) + .ok_or_else(|| { + format!( + "Legacy project filename is not valid UTF-8: {}", + canonical_path.display() + ) + })? + .to_string(); + validate_stored_project_id(&id)?; + if !ids.insert(id.clone()) { + return Err(format!("Duplicate legacy project id: {id}")); + } + + let fingerprint = hex::encode(Sha256::digest(&bytes)); + let backup_path = publish_backup(&backup_dir, &fingerprint, &bytes, &mut backup_bytes)?; + candidates.push(LegacyProject { + source_path: canonical_path.to_string_lossy().into_owned(), + fingerprint, + backup_path: backup_path.to_string_lossy().into_owned(), + project: parse_legacy_project(&id, raw)?, + }); + } + sync_directory(&backup_dir)?; + Ok(LegacyScan { + source_present: true, + source_root: Some(source_root.to_string_lossy().into_owned()), + projects: candidates, + }) +} + +fn read_bounded_regular_file(root: &Path, path: &Path) -> Result<(Vec, PathBuf), String> { + let metadata = fs::symlink_metadata(path).map_err(|error| { + format!( + "Failed to inspect legacy project {}: {error}", + path.display() + ) + })?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(format!( + "Legacy project must be a regular non-symlink file: {}", + path.display() + )); + } + if metadata.len() > MAX_PROJECT_FILE_BYTES { + return Err(format!( + "Legacy project exceeds the {MAX_PROJECT_FILE_BYTES}-byte safety limit: {}", + path.display() + )); + } + + let canonical_path = path.canonicalize().map_err(|error| { + format!( + "Failed to resolve legacy project {}: {error}", + path.display() + ) + })?; + if canonical_path.parent() != Some(root) { + return Err(format!( + "Legacy project escaped the fixed projects root: {}", + canonical_path.display() + )); + } + + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW); + } + let file = options.open(&canonical_path).map_err(|error| { + format!( + "Failed to open legacy project {} without following links: {error}", + canonical_path.display() + ) + })?; + let opened_metadata = file.metadata().map_err(|error| { + format!( + "Failed to inspect opened legacy project {}: {error}", + canonical_path.display() + ) + })?; + if !opened_metadata.is_file() || opened_metadata.len() > MAX_PROJECT_FILE_BYTES { + return Err(format!( + "Legacy project changed while being scanned: {}", + canonical_path.display() + )); + } + let mut bytes = Vec::with_capacity(opened_metadata.len() as usize); + file.take(MAX_PROJECT_FILE_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|error| { + format!( + "Failed to read legacy project {}: {error}", + canonical_path.display() + ) + })?; + if bytes.len() as u64 > MAX_PROJECT_FILE_BYTES { + return Err(format!( + "Legacy project grew beyond the safety limit while being scanned: {}", + canonical_path.display() + )); + } + Ok((bytes, canonical_path)) +} + +fn parse_legacy_project(id: &str, raw: &str) -> Result { + let (frontmatter, prompt) = parse_legacy_frontmatter(raw); + let mut properties = frontmatter.properties; + let name = if frontmatter.name.is_empty() { + id.to_string() + } else { + frontmatter.name + }; + if name != id { + properties.insert("title".to_string(), Value::String(name.clone())); + } + + let workspaces = parse_workspaces(&properties)?; + Ok(StoredProject { + id: id.to_string(), + name, + description: frontmatter.description, + prompt, + icon: string_property(&properties, "icon").unwrap_or_default(), + color: string_property(&properties, "color").unwrap_or_default(), + use_worktrees: properties + .get("useWorktrees") + .and_then(Value::as_bool) + .unwrap_or(false), + order: properties.get("order").and_then(Value::as_i64).unwrap_or(0), + archived_at: string_property(&properties, "archivedAt"), + artifact_json: optional_json_property(&properties, "artifact")?, + chat_groups_json: optional_json_property(&properties, "chatGroups")?, + properties_json: serde_json::to_string(&properties) + .map_err(|error| format!("Failed to preserve project properties: {error}"))?, + workspaces, + }) +} + +fn parse_legacy_frontmatter(raw: &str) -> (LegacyFrontmatter, String) { + if !raw.trim_start().starts_with("---") { + return (LegacyFrontmatter::default(), raw.to_string()); + } + let parts: Vec<&str> = raw.split("---").collect(); + if parts.len() < 3 { + return (LegacyFrontmatter::default(), raw.to_string()); + } + match yaml_serde::from_str::(parts[1].trim()) { + Ok(frontmatter) => (frontmatter, parts[2..].join("---").trim().to_string()), + Err(_) => (LegacyFrontmatter::default(), raw.to_string()), + } +} + +fn parse_workspaces(properties: &BTreeMap) -> Result, String> { + if let Some(value) = properties.get("projectWorkspaces") { + let workspaces: Vec = serde_json::from_value(value.clone()) + .map_err(|error| format!("Invalid projectWorkspaces metadata: {error}"))?; + let mut stored = Vec::with_capacity(workspaces.len()); + for workspace in workspaces { + let Some(workspace) = workspace.as_object() else { + continue; + }; + let Some(path) = workspace + .get("path") + .and_then(Value::as_str) + .map(str::trim) + .filter(|path| !path.is_empty()) + .map(str::to_string) + else { + continue; + }; + let id = workspace + .get("id") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + .map(str::to_string) + .unwrap_or_else(|| workspace_id(&path)); + stored.push(StoredWorkspace { + id, + path, + kind: valid_workspace_kind(workspace.get("kind")), + source: valid_workspace_source(workspace.get("source")), + branch: workspace + .get("branch") + .and_then(Value::as_str) + .map(str::to_string), + repository_path: trimmed_string(workspace.get("repositoryPath")), + worktree_path: trimmed_string(workspace.get("worktreePath")), + startup_mode: valid_startup_mode(workspace.get("startupMode")), + }); + } + let stored = dedupe_workspaces(stored); + if !stored.is_empty() { + return Ok(stored); + } + } + + let working_dirs: Vec = properties + .get("workingDirs") + .cloned() + .map(serde_json::from_value) + .transpose() + .map_err(|error| format!("Invalid workingDirs metadata: {error}"))? + .unwrap_or_default(); + let startup_mode = if properties + .get("useWorktrees") + .and_then(Value::as_bool) + .unwrap_or(false) + { + "auto-worktree" + } else { + "none" + }; + Ok(dedupe_workspaces( + working_dirs + .into_iter() + .filter_map(|path| { + let path = path.trim().to_string(); + (!path.is_empty()).then(|| StoredWorkspace { + id: workspace_id(&path), + path, + kind: "directory".to_string(), + source: "inferred".to_string(), + branch: None, + repository_path: None, + worktree_path: None, + startup_mode: startup_mode.to_string(), + }) + }) + .collect(), + )) +} + +fn workspace_id(path: &str) -> String { + format!("path:{}", workspace_identity_key(path)) +} + +fn dedupe_workspaces(workspaces: Vec) -> Vec { + let mut identities = Vec::::new(); + let mut deduped = Vec::::new(); + for workspace in workspaces { + let identity = workspace_identity_key(&workspace.path); + if let Some(position) = identities.iter().position(|existing| existing == &identity) { + deduped[position] = workspace; + } else { + identities.push(identity); + deduped.push(workspace); + } + } + deduped +} + +fn valid_workspace_kind(value: Option<&Value>) -> String { + match value.and_then(Value::as_str) { + Some( + value @ ("repository" + | "git-main-worktree" + | "git-linked-worktree" + | "git-detached-checkout" + | "subdirectory" + | "non-git-directory"), + ) => value.to_string(), + _ => "directory".to_string(), + } +} + +fn valid_workspace_source(value: Option<&Value>) -> String { + match value.and_then(Value::as_str) { + Some(value @ ("selected" | "created" | "excluded" | "inferred")) => value.to_string(), + _ => "inferred".to_string(), + } +} + +fn valid_startup_mode(value: Option<&Value>) -> String { + match value.and_then(Value::as_str) { + Some("worktree") => "auto-worktree".to_string(), + Some("branch") => "ask-worktree".to_string(), + Some(value @ ("ask-worktree" | "auto-worktree")) => value.to_string(), + _ => "none".to_string(), + } +} + +fn trimmed_string(value: Option<&Value>) -> Option { + value + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) +} + +fn workspace_identity_key(path: &str) -> String { + let normalized = normalize_lexical_path(path); + if is_windows_absolute_path(path) || cfg!(target_os = "macos") { + normalized.to_lowercase() + } else { + normalized + } +} + +fn is_windows_absolute_path(path: &str) -> bool { + let path = path.trim().replace('\\', "/"); + let bytes = path.as_bytes(); + (bytes.len() >= 3 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':' && bytes[2] == b'/') + || windows_unc_root_parts(&path).is_some_and(|(server, share, _)| { + !matches!(server, "." | "..") && !matches!(share, "." | "..") + }) +} + +fn windows_unc_root_parts(path: &str) -> Option<(&str, &str, &str)> { + let unc = path.strip_prefix("//")?; + let mut parts = unc.splitn(3, '/'); + let server = parts.next()?; + let share = parts.next()?; + if server.is_empty() || share.is_empty() || server == "." || share == ".." { + return None; + } + Some((server, share, parts.next().unwrap_or_default())) +} + +fn normalize_lexical_path(path: &str) -> String { + let mut path = path.trim().replace('\\', "/"); + let is_drive_root = path.len() == 3 + && path.as_bytes()[0].is_ascii_alphabetic() + && path.as_bytes()[1] == b':' + && path.as_bytes()[2] == b'/'; + if path != "/" && !is_drive_root { + while path.ends_with('/') { + path.pop(); + } + } + + let bytes = path.as_bytes(); + let (prefix, remainder) = if bytes.len() >= 3 + && bytes[0].is_ascii_alphabetic() + && bytes[1] == b':' + && bytes[2] == b'/' + { + (path[..3].to_string(), path[3..].to_string()) + } else if bytes.len() >= 2 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':' { + (path[..2].to_string(), path[2..].to_string()) + } else if let Some((server, share, remainder)) = windows_unc_root_parts(&path) { + (format!("//{server}/{share}"), remainder.to_string()) + } else if let Some(remainder) = path.strip_prefix('/') { + ("/".to_string(), remainder.to_string()) + } else { + (String::new(), path) + }; + let clamps_parent = prefix == "/" || prefix.ends_with('/') || prefix.starts_with("//"); + let mut segments: Vec<&str> = Vec::new(); + for segment in remainder.split('/') { + match segment { + "" | "." => {} + ".." if segments.last().is_some_and(|last| *last != "..") => { + segments.pop(); + } + ".." if !clamps_parent => segments.push(segment), + ".." => {} + _ => segments.push(segment), + } + } + let joined = segments.join("/"); + match (prefix.as_str(), joined.is_empty()) { + ("", _) => joined, + ("/", true) => "/".to_string(), + (_, true) => prefix, + (_, false) if prefix.ends_with('/') || is_drive_relative_prefix(&prefix) => { + format!("{prefix}{joined}") + } + (_, false) => format!("{prefix}/{joined}"), + } +} + +fn is_drive_relative_prefix(prefix: &str) -> bool { + let bytes = prefix.as_bytes(); + bytes.len() == 2 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':' +} + +fn string_property(properties: &BTreeMap, key: &str) -> Option { + properties + .get(key) + .and_then(Value::as_str) + .map(str::to_string) +} + +fn optional_json_property( + properties: &BTreeMap, + key: &str, +) -> Result, String> { + properties + .get(key) + .filter(|value| !value.is_null()) + .map(serde_json::to_string) + .transpose() + .map_err(|error| format!("Failed to preserve project {key}: {error}")) +} + +fn validate_stored_project_id(id: &str) -> Result<(), String> { + if id.is_empty() || id.len() > 255 { + return Err(format!("Invalid legacy project id: {id:?}")); + } + Ok(()) +} + +fn prepare_backup_directory( + backup_dir: &Path, + source_root: &Path, +) -> Result<(PathBuf, u64), String> { + let anchor = backup_dir + .parent() + .and_then(Path::parent) + .ok_or_else(|| format!("Invalid project backup path: {}", backup_dir.display()))?; + let canonical_anchor = anchor.canonicalize().map_err(|error| { + format!( + "Failed to resolve Berd app data directory {}: {error}", + anchor.display() + ) + })?; + let relative = backup_dir.strip_prefix(anchor).map_err(|_| { + format!( + "Project backup path escaped Berd app data: {}", + backup_dir.display() + ) + })?; + let mut current = canonical_anchor.clone(); + for component in relative.components() { + let std::path::Component::Normal(component) = component else { + return Err(format!( + "Project backup path contains an unsafe component: {}", + backup_dir.display() + )); + }; + current.push(component); + match fs::symlink_metadata(¤t) { + Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_dir() => { + return Err(format!( + "Project backup path component must be a non-symlink directory: {}", + current.display() + )); + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + fs::create_dir(¤t).map_err(|error| { + format!( + "Failed to create project backup directory {}: {error}", + current.display() + ) + })?; + } + Err(error) => { + return Err(format!( + "Failed to inspect project backup directory {}: {error}", + current.display() + )); + } + } + } + let canonical_backup = current.canonicalize().map_err(|error| { + format!( + "Failed to resolve project backup directory {}: {error}", + current.display() + ) + })?; + if !canonical_backup.starts_with(&canonical_anchor) + || canonical_backup.starts_with(source_root) + || source_root.starts_with(&canonical_backup) + { + return Err(format!( + "Project backup directory escaped its fixed Berd app-data root: {}", + canonical_backup.display() + )); + } + + let mut total_bytes = 0_u64; + for entry in fs::read_dir(&canonical_backup).map_err(|error| { + format!( + "Failed to inspect project backup directory {}: {error}", + canonical_backup.display() + ) + })? { + let entry = entry.map_err(|error| format!("Failed to inspect project backup: {error}"))?; + let metadata = fs::symlink_metadata(entry.path()).map_err(|error| { + format!( + "Failed to inspect project backup {}: {error}", + entry.path().display() + ) + })?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(format!( + "Project backup entries must be regular non-symlink files: {}", + entry.path().display() + )); + } + total_bytes = total_bytes + .checked_add(metadata.len()) + .ok_or_else(|| "Project backup size overflowed".to_string())?; + } + if total_bytes > MAX_BACKUP_BYTES { + return Err(format!( + "Project backups exceed the {MAX_BACKUP_BYTES}-byte safety limit" + )); + } + Ok((canonical_backup, total_bytes)) +} + +fn publish_backup( + backup_dir: &Path, + fingerprint: &str, + bytes: &[u8], + total_bytes: &mut u64, +) -> Result { + let path = backup_dir.join(format!("{fingerprint}.md")); + if path.exists() { + verify_backup(&path, bytes)?; + return Ok(path); + } + let next_total = total_bytes + .checked_add(bytes.len() as u64) + .ok_or_else(|| "Project backup size overflowed".to_string())?; + if next_total > MAX_BACKUP_BYTES { + return Err(format!( + "Project backups would exceed the {MAX_BACKUP_BYTES}-byte safety limit" + )); + } + + let temporary = backup_dir.join(format!(".{fingerprint}.{}.tmp", uuid::Uuid::new_v4())); + let mut options = OpenOptions::new(); + options.create_new(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW); + } + let mut file = options.open(&temporary).map_err(|error| { + format!( + "Failed to stage legacy project backup {}: {error}", + temporary.display() + ) + })?; + file.write_all(bytes).map_err(|error| { + format!( + "Failed to write legacy project backup {}: {error}", + temporary.display() + ) + })?; + file.sync_all().map_err(|error| { + format!( + "Failed to sync legacy project backup {}: {error}", + temporary.display() + ) + })?; + if let Err(error) = fs::hard_link(&temporary, &path) { + if path.exists() { + verify_backup(&path, bytes)?; + let _ = fs::remove_file(&temporary); + } else { + let _ = fs::remove_file(&temporary); + return Err(format!( + "Failed to publish legacy project backup {}: {error}", + path.display() + )); + } + } else { + fs::remove_file(&temporary).map_err(|error| { + format!( + "Failed to remove staged project backup {}: {error}", + temporary.display() + ) + })?; + *total_bytes = next_total; + } + Ok(path) +} + +fn verify_backup(path: &Path, bytes: &[u8]) -> Result<(), String> { + let metadata = fs::symlink_metadata(path).map_err(|error| { + format!( + "Failed to inspect project backup {}: {error}", + path.display() + ) + })?; + if metadata.file_type().is_symlink() + || !metadata.is_file() + || metadata.len() > MAX_PROJECT_FILE_BYTES + { + return Err(format!( + "Project backup must be a bounded regular non-symlink file: {}", + path.display() + )); + } + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW); + } + let file = options.open(path).map_err(|error| { + format!( + "Failed to verify project backup {}: {error}", + path.display() + ) + })?; + let mut existing = Vec::with_capacity(metadata.len() as usize); + file.take(MAX_PROJECT_FILE_BYTES + 1) + .read_to_end(&mut existing) + .map_err(|error| { + format!( + "Failed to verify project backup {}: {error}", + path.display() + ) + })?; + if Sha256::digest(existing) != Sha256::digest(bytes) { + return Err(format!( + "Project backup fingerprint collision at {}", + path.display() + )); + } + Ok(()) +} + +#[cfg(unix)] +fn sync_directory(path: &Path) -> Result<(), String> { + File::open(path) + .and_then(|directory| directory.sync_all()) + .map_err(|error| format!("Failed to sync directory {}: {error}", path.display())) +} + +#[cfg(not(unix))] +fn sync_directory(_path: &Path) -> Result<(), String> { + Ok(()) +} + +async fn apply_shadow_import( + pool: &sqlx::SqlitePool, + scan: LegacyScan, + expected_generation: i64, +) -> Result { + let mut transaction = pool.begin_with("BEGIN IMMEDIATE").await.map_err(db_error)?; + ensure_supported_migration_version_in_transaction(&mut transaction).await?; + let state = sqlx::query( + "SELECT phase, shadow_scan_generation FROM project_storage_state WHERE singleton_id = 1", + ) + .fetch_one(&mut *transaction) + .await + .map_err(db_error)?; + let phase: String = state.get("phase"); + if phase == "native" { + return Err( + "Legacy project import is disabled after native storage activation".to_string(), + ); + } + if state.get::("shadow_scan_generation") != expected_generation { + return Ok(ShadowImportReport { + phase, + scan_generation: expected_generation, + source_count: scan.projects.len(), + imported: 0, + updated: 0, + removed: 0, + unchanged: 0, + superseded: true, + }); + } + + let previous_migration = + sqlx::query("SELECT checkpoint_json FROM project_migration_runs WHERE migration_key = ?") + .bind(MIGRATION_KEY) + .fetch_optional(&mut *transaction) + .await + .map_err(db_error)?; + let previous_checkpoint = previous_migration + .as_ref() + .map(|row| row.get::("checkpoint_json")); + let previous_root = previous_checkpoint + .as_deref() + .and_then(|checkpoint| serde_json::from_str::(checkpoint).ok()) + .and_then(|checkpoint| { + checkpoint + .get("sourceRoot") + .and_then(Value::as_str) + .map(str::to_string) + }); + if let (Some(previous_root), Some(current_root)) = (&previous_root, &scan.source_root) { + if previous_root != current_root { + return Err(format!( + "Legacy projects root changed from {previous_root} to {current_root}; refusing to replace the existing shadow mirror" + )); + } + } + + let now = Utc::now().to_rfc3339(); + let mut imported = 0; + let mut updated = 0; + let mut unchanged = 0; + let mut source_paths = HashSet::new(); + for candidate in &scan.projects { + source_paths.insert(candidate.source_path.clone()); + let receipt = sqlx::query( + "SELECT source_fingerprint, project_id FROM legacy_project_imports WHERE source_path = ?", + ) + .bind(&candidate.source_path) + .fetch_optional(&mut *transaction) + .await + .map_err(db_error)?; + match receipt { + Some(row) + if row.get::("source_fingerprint") == candidate.fingerprint + && row.get::("project_id") == candidate.project.id => + { + unchanged += 1; + continue; + } + Some(row) if row.get::("project_id") != candidate.project.id => { + return Err(format!( + "Legacy source {} changed project identity", + candidate.source_path + )); + } + Some(_) => updated += 1, + None => { + let owner: Option = sqlx::query_scalar( + "SELECT source_path FROM legacy_project_imports WHERE project_id = ?", + ) + .bind(&candidate.project.id) + .fetch_optional(&mut *transaction) + .await + .map_err(db_error)?; + if let Some(owner) = owner { + return Err(format!( + "Legacy project id {} is already owned by {}", + candidate.project.id, owner + )); + } + imported += 1; + } + } + upsert_project(&mut transaction, &candidate.project, &now).await?; + sqlx::query( + "INSERT INTO legacy_project_imports \ + (source_path, source_fingerprint, project_id, backup_path, imported_at) \ + VALUES (?, ?, ?, ?, ?) \ + ON CONFLICT(source_path) DO UPDATE SET \ + source_fingerprint = excluded.source_fingerprint, \ + project_id = excluded.project_id, backup_path = excluded.backup_path, \ + imported_at = excluded.imported_at", + ) + .bind(&candidate.source_path) + .bind(&candidate.fingerprint) + .bind(&candidate.project.id) + .bind(&candidate.backup_path) + .bind(&now) + .execute(&mut *transaction) + .await + .map_err(db_error)?; + } + + let mut removed = 0; + if scan.source_present { + let receipts = sqlx::query("SELECT source_path, project_id FROM legacy_project_imports") + .fetch_all(&mut *transaction) + .await + .map_err(db_error)?; + for receipt in receipts { + let source_path: String = receipt.get("source_path"); + if source_paths.contains(&source_path) { + continue; + } + let project_id: String = receipt.get("project_id"); + sqlx::query("DELETE FROM legacy_project_imports WHERE source_path = ?") + .bind(&source_path) + .execute(&mut *transaction) + .await + .map_err(db_error)?; + sqlx::query( + "DELETE FROM session_projects WHERE project_id = ? AND source = 'legacy-backfill'", + ) + .bind(&project_id) + .execute(&mut *transaction) + .await + .map_err(db_error)?; + sqlx::query("DELETE FROM projects WHERE id = ?") + .bind(&project_id) + .execute(&mut *transaction) + .await + .map_err(db_error)?; + removed += 1; + } + } + + let changed = imported > 0 || updated > 0 || removed > 0; + let checkpoint_root = scan.source_root.as_ref().or(previous_root.as_ref()); + let checkpoint = serde_json::json!({ + "scanGeneration": expected_generation, + "sourcePresent": scan.source_present, + "sourceRoot": checkpoint_root, + "sourceCount": scan.projects.len(), + "imported": imported, + "updated": updated, + "removed": removed, + "unchanged": unchanged, + }); + sqlx::query( + "INSERT INTO project_migration_runs \ + (migration_key, version, scan_generation, state, checkpoint_json, completed_at, last_error, updated_at) \ + VALUES (?, ?, ?, 'complete', ?, ?, NULL, ?) \ + ON CONFLICT(migration_key) DO UPDATE SET \ + scan_generation = excluded.scan_generation, state = 'complete', \ + checkpoint_json = excluded.checkpoint_json, \ + completed_at = excluded.completed_at, last_error = NULL, \ + updated_at = excluded.updated_at", + ) + .bind(MIGRATION_KEY) + .bind(MIGRATION_VERSION) + .bind(expected_generation) + .bind(checkpoint.to_string()) + .bind(&now) + .bind(&now) + .execute(&mut *transaction) + .await + .map_err(db_error)?; + if changed || phase == "legacy" { + sqlx::query( + "UPDATE project_storage_state SET phase = 'shadow', \ + collection_revision = collection_revision + 1, shadow_completed_at = ? \ + WHERE singleton_id = 1 AND phase IN ('legacy', 'shadow')", + ) + .bind(&now) + .execute(&mut *transaction) + .await + .map_err(db_error)?; + } + transaction.commit().await.map_err(db_error)?; + + Ok(ShadowImportReport { + phase: "shadow".to_string(), + scan_generation: expected_generation, + source_count: scan.projects.len(), + imported, + updated, + removed, + unchanged, + superseded: false, + }) +} + +async fn upsert_project( + transaction: &mut Transaction<'_, Sqlite>, + project: &StoredProject, + now: &str, +) -> Result<(), String> { + sqlx::query( + "INSERT INTO projects \ + (id, name, description, prompt, icon, color, use_worktrees, order_index, \ + archived_at, artifact_json, chat_groups_json, extra_properties_json, revision, \ + created_at, updated_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?) \ + ON CONFLICT(id) DO UPDATE SET \ + name = excluded.name, description = excluded.description, prompt = excluded.prompt, \ + icon = excluded.icon, color = excluded.color, use_worktrees = excluded.use_worktrees, \ + order_index = excluded.order_index, archived_at = excluded.archived_at, \ + artifact_json = excluded.artifact_json, chat_groups_json = excluded.chat_groups_json, \ + extra_properties_json = excluded.extra_properties_json, \ + revision = projects.revision + 1, updated_at = excluded.updated_at", + ) + .bind(&project.id) + .bind(&project.name) + .bind(&project.description) + .bind(&project.prompt) + .bind(&project.icon) + .bind(&project.color) + .bind(project.use_worktrees) + .bind(project.order) + .bind(&project.archived_at) + .bind(&project.artifact_json) + .bind(&project.chat_groups_json) + .bind(&project.properties_json) + .bind(now) + .bind(now) + .execute(&mut **transaction) + .await + .map_err(db_error)?; + + sqlx::query("DELETE FROM project_workspaces WHERE project_id = ?") + .bind(&project.id) + .execute(&mut **transaction) + .await + .map_err(db_error)?; + for (position, workspace) in project.workspaces.iter().enumerate() { + sqlx::query( + "INSERT INTO project_workspaces \ + (project_id, position, workspace_id, path, kind, source, branch, repository_path, \ + worktree_path, startup_mode) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(&project.id) + .bind(position as i64) + .bind(&workspace.id) + .bind(&workspace.path) + .bind(&workspace.kind) + .bind(&workspace.source) + .bind(&workspace.branch) + .bind(&workspace.repository_path) + .bind(&workspace.worktree_path) + .bind(&workspace.startup_mode) + .execute(&mut **transaction) + .await + .map_err(db_error)?; + } + Ok(()) +} + +async fn backfill_legacy_association( + pool: &sqlx::SqlitePool, + request: LegacySessionProjectBackfillRequest, +) -> Result { + let mut transaction = pool.begin_with("BEGIN IMMEDIATE").await.map_err(db_error)?; + require_phase_one_of(&mut transaction, &["shadow", "native"]).await?; + let existing = read_session_association_in_transaction( + &mut transaction, + &request.session_backend_id, + &request.session_id, + ) + .await?; + if existing.has_association { + transaction.commit().await.map_err(db_error)?; + return Ok(existing); + } + require_existing_project(&mut transaction, Some(&request.project_id)).await?; + let now = Utc::now().to_rfc3339(); + sqlx::query( + "INSERT OR IGNORE INTO session_projects \ + (session_backend_id, session_id, project_id, source, revision, updated_at) \ + VALUES (?, ?, ?, 'legacy-backfill', 1, ?)", + ) + .bind(&request.session_backend_id) + .bind(&request.session_id) + .bind(&request.project_id) + .bind(now) + .execute(&mut *transaction) + .await + .map_err(db_error)?; + let association = read_session_association_in_transaction( + &mut transaction, + &request.session_backend_id, + &request.session_id, + ) + .await?; + transaction.commit().await.map_err(db_error)?; + Ok(association) +} + +async fn update_native_association( + pool: &sqlx::SqlitePool, + request: UpdateSessionProjectRequest, +) -> Result { + let mut transaction = pool.begin_with("BEGIN IMMEDIATE").await.map_err(db_error)?; + require_phase(&mut transaction, "native").await?; + require_existing_project(&mut transaction, request.project_id.as_deref()).await?; + let now = Utc::now().to_rfc3339(); + + let changed = if request.expected_revision == 0 { + sqlx::query( + "INSERT OR IGNORE INTO session_projects \ + (session_backend_id, session_id, project_id, source, revision, updated_at) \ + VALUES (?, ?, ?, 'native', 1, ?)", + ) + .bind(&request.session_backend_id) + .bind(&request.session_id) + .bind(&request.project_id) + .bind(&now) + .execute(&mut *transaction) + .await + .map_err(db_error)? + .rows_affected() + } else { + sqlx::query( + "UPDATE session_projects SET project_id = ?, source = 'native', \ + revision = revision + 1, updated_at = ? \ + WHERE session_backend_id = ? AND session_id = ? AND revision = ?", + ) + .bind(&request.project_id) + .bind(&now) + .bind(&request.session_backend_id) + .bind(&request.session_id) + .bind(request.expected_revision) + .execute(&mut *transaction) + .await + .map_err(db_error)? + .rows_affected() + }; + if changed != 1 { + let current = read_session_association_in_transaction( + &mut transaction, + &request.session_backend_id, + &request.session_id, + ) + .await?; + return Err(format!( + "Session project revision conflict; current revision is {}", + current + .revision + .map(|revision| revision.to_string()) + .unwrap_or_else(|| "absent".to_string()) + )); + } + + let association = read_session_association_in_transaction( + &mut transaction, + &request.session_backend_id, + &request.session_id, + ) + .await?; + transaction.commit().await.map_err(db_error)?; + Ok(association) +} + +async fn read_session_association( + pool: &sqlx::SqlitePool, + session_backend_id: &str, + session_id: &str, +) -> Result { + let row = sqlx::query( + "SELECT project_id, revision FROM session_projects \ + WHERE session_backend_id = ? AND session_id = ?", + ) + .bind(session_backend_id) + .bind(session_id) + .fetch_optional(pool) + .await + .map_err(db_error)?; + Ok(association_from_row(session_backend_id, session_id, row)) +} + +async fn read_session_association_in_transaction( + transaction: &mut Transaction<'_, Sqlite>, + session_backend_id: &str, + session_id: &str, +) -> Result { + let row = sqlx::query( + "SELECT project_id, revision FROM session_projects \ + WHERE session_backend_id = ? AND session_id = ?", + ) + .bind(session_backend_id) + .bind(session_id) + .fetch_optional(&mut **transaction) + .await + .map_err(db_error)?; + Ok(association_from_row(session_backend_id, session_id, row)) +} + +fn association_from_row( + session_backend_id: &str, + session_id: &str, + row: Option, +) -> SessionProjectAssociation { + SessionProjectAssociation { + session_backend_id: session_backend_id.to_string(), + session_id: session_id.to_string(), + has_association: row.is_some(), + project_id: row + .as_ref() + .and_then(|row| row.get::, _>("project_id")), + revision: row.map(|row| row.get("revision")), + } +} + +async fn require_phase( + transaction: &mut Transaction<'_, Sqlite>, + required: &str, +) -> Result<(), String> { + require_phase_one_of(transaction, &[required]).await +} + +async fn require_phase_one_of( + transaction: &mut Transaction<'_, Sqlite>, + allowed: &[&str], +) -> Result<(), String> { + let phase: String = + sqlx::query_scalar("SELECT phase FROM project_storage_state WHERE singleton_id = 1") + .fetch_one(&mut **transaction) + .await + .map_err(db_error)?; + if !allowed.contains(&phase.as_str()) { + return Err(format!( + "Project storage phase must be {}; current phase is {phase}", + allowed.join(" or ") + )); + } + Ok(()) +} + +async fn require_existing_project( + transaction: &mut Transaction<'_, Sqlite>, + project_id: Option<&str>, +) -> Result<(), String> { + let Some(project_id) = project_id else { + return Ok(()); + }; + let exists: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM projects WHERE id = ?)") + .bind(project_id) + .fetch_one(&mut **transaction) + .await + .map_err(db_error)?; + if !exists { + return Err(format!("Project {project_id:?} does not exist")); + } + Ok(()) +} + +async fn ensure_supported_migration_version(pool: &sqlx::SqlitePool) -> Result<(), String> { + let version: Option = + sqlx::query_scalar("SELECT version FROM project_migration_runs WHERE migration_key = ?") + .bind(MIGRATION_KEY) + .fetch_optional(pool) + .await + .map_err(db_error)?; + reject_future_migration_version(version) +} + +async fn ensure_supported_migration_version_in_transaction( + transaction: &mut Transaction<'_, Sqlite>, +) -> Result<(), String> { + let version: Option = + sqlx::query_scalar("SELECT version FROM project_migration_runs WHERE migration_key = ?") + .bind(MIGRATION_KEY) + .fetch_optional(&mut **transaction) + .await + .map_err(db_error)?; + reject_future_migration_version(version) +} + +fn reject_future_migration_version(version: Option) -> Result<(), String> { + if let Some(version) = version.filter(|version| *version > MIGRATION_VERSION) { + return Err(format!( + "Project migration version {version} is newer than this Berd build supports ({MIGRATION_VERSION}); refusing to modify it" + )); + } + Ok(()) +} + +fn validate_session_key(session_backend_id: &str, session_id: &str) -> Result<(), String> { + if session_backend_id.trim().is_empty() + || session_backend_id.trim() != session_backend_id + || session_backend_id.len() > 128 + { + return Err("Session backend id must contain 1 to 128 bytes".to_string()); + } + if session_id.trim().is_empty() || session_id.trim() != session_id || session_id.len() > 512 { + return Err("Session id must contain 1 to 512 bytes".to_string()); + } + Ok(()) +} + +async fn record_import_failure( + pool: &sqlx::SqlitePool, + expected_generation: i64, + error: &str, +) -> Result { + let mut transaction = pool.begin_with("BEGIN IMMEDIATE").await.map_err(db_error)?; + ensure_supported_migration_version_in_transaction(&mut transaction).await?; + let state = sqlx::query( + "SELECT phase, shadow_scan_generation FROM project_storage_state WHERE singleton_id = 1", + ) + .fetch_one(&mut *transaction) + .await + .map_err(db_error)?; + if state.get::("phase") == "native" + || state.get::("shadow_scan_generation") != expected_generation + { + transaction.commit().await.map_err(db_error)?; + return Ok(false); + } + let now = Utc::now().to_rfc3339(); + sqlx::query( + "INSERT INTO project_migration_runs \ + (migration_key, version, scan_generation, state, checkpoint_json, completed_at, last_error, updated_at) \ + VALUES (?, ?, ?, 'failed', '{}', NULL, ?, ?) \ + ON CONFLICT(migration_key) DO UPDATE SET \ + scan_generation = excluded.scan_generation, state = 'failed', completed_at = NULL, \ + last_error = excluded.last_error, \ + updated_at = excluded.updated_at \ + WHERE project_migration_runs.version <= excluded.version", + ) + .bind(MIGRATION_KEY) + .bind(MIGRATION_VERSION) + .bind(expected_generation) + .bind(error) + .bind(now) + .execute(&mut *transaction) + .await + .map_err(db_error)?; + transaction.commit().await.map_err(db_error)?; + Ok(true) +} + +fn db_error(error: sqlx::Error) -> String { + format!("Project storage database operation failed: {error}") +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + fn project_markdown(name: &str, prompt: &str) -> String { + format!( + "---\nname: {name}\ndescription: test project\nicon: wrench\norder: 7\nunknownProperty:\n nested: true\nworkingDirs:\n - /tmp/work\n---\n\n{prompt}\n" + ) + } + + async fn state() -> (tempfile::TempDir, BerdPersistenceState) { + let dir = tempdir().unwrap(); + let state = BerdPersistenceState::new_for_tests(dir.path().join("test.sqlite")) + .await + .unwrap(); + (dir, state) + } + + async fn activate_for_test(state: &BerdPersistenceState) { + sqlx::query("UPDATE project_storage_state SET phase = 'shadow' WHERE singleton_id = 1") + .execute(state.pool()) + .await + .unwrap(); + sqlx::query("UPDATE project_storage_state SET phase = 'native' WHERE singleton_id = 1") + .execute(state.pool()) + .await + .unwrap(); + } + + async fn apply_scan( + state: &BerdPersistenceState, + scan: LegacyScan, + ) -> Result { + let generation = claim_shadow_scan(state.pool()).await?; + apply_shadow_import(state.pool(), scan, generation).await + } + + #[tokio::test] + async fn shadow_import_is_idempotent_and_preserves_unknown_properties() { + let (db_dir, state) = state().await; + let source = tempdir().unwrap(); + let backup = db_dir.path().join("backups"); + fs::write( + source.path().join("alpha.md"), + project_markdown("Alpha", "Always test."), + ) + .unwrap(); + + let first = scan_legacy_projects(source.path(), &backup).unwrap(); + let report = apply_scan(&state, first).await.unwrap(); + assert_eq!(report.imported, 1); + assert_eq!(report.unchanged, 0); + + let second = scan_legacy_projects(source.path(), &backup).unwrap(); + let report = apply_scan(&state, second).await.unwrap(); + assert_eq!(report.imported, 0); + assert_eq!(report.unchanged, 1); + + let row = sqlx::query( + "SELECT name, prompt, extra_properties_json FROM projects WHERE id = 'alpha'", + ) + .fetch_one(state.pool()) + .await + .unwrap(); + assert_eq!(row.get::("name"), "Alpha"); + assert_eq!(row.get::("prompt"), "Always test."); + let properties: Value = + serde_json::from_str(&row.get::("extra_properties_json")).unwrap(); + assert_eq!(properties["unknownProperty"]["nested"], true); + assert_eq!(properties["title"], "Alpha"); + assert_eq!(fs::read_dir(backup).unwrap().count(), 1); + } + + #[tokio::test] + async fn shadow_rescan_updates_changed_sources_and_removes_deleted_sources() { + let (_db_dir, state) = state().await; + let source = tempdir().unwrap(); + let backup = tempdir().unwrap(); + let project_path = source.path().join("alpha.md"); + fs::write(&project_path, project_markdown("Alpha", "First")).unwrap(); + apply_scan( + &state, + scan_legacy_projects(source.path(), backup.path()).unwrap(), + ) + .await + .unwrap(); + + fs::write(&project_path, project_markdown("Alpha", "Second")).unwrap(); + let report = apply_scan( + &state, + scan_legacy_projects(source.path(), backup.path()).unwrap(), + ) + .await + .unwrap(); + assert_eq!(report.updated, 1); + assert_eq!( + sqlx::query_scalar::<_, String>("SELECT prompt FROM projects WHERE id = 'alpha'") + .fetch_one(state.pool()) + .await + .unwrap(), + "Second" + ); + + fs::remove_file(project_path).unwrap(); + let report = apply_scan( + &state, + scan_legacy_projects(source.path(), backup.path()).unwrap(), + ) + .await + .unwrap(); + assert_eq!(report.removed, 1); + assert_eq!( + sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM projects") + .fetch_one(state.pool()) + .await + .unwrap(), + 0 + ); + } + + #[tokio::test] + async fn missing_or_changed_legacy_root_cannot_erase_an_existing_mirror() { + let (_db_dir, state) = state().await; + let source = tempdir().unwrap(); + let backup = tempdir().unwrap(); + fs::write( + source.path().join("alpha.md"), + project_markdown("Alpha", "Prompt"), + ) + .unwrap(); + apply_scan( + &state, + scan_legacy_projects(source.path(), backup.path()).unwrap(), + ) + .await + .unwrap(); + + let missing = apply_scan( + &state, + LegacyScan { + source_present: false, + source_root: None, + projects: Vec::new(), + }, + ) + .await + .unwrap(); + assert_eq!(missing.removed, 0); + assert_eq!( + sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM projects") + .fetch_one(state.pool()) + .await + .unwrap(), + 1 + ); + + let different_root = tempdir().unwrap(); + let error = apply_scan( + &state, + scan_legacy_projects(different_root.path(), backup.path()).unwrap(), + ) + .await + .unwrap_err(); + assert!(error.contains("root changed")); + } + + #[cfg(unix)] + #[test] + fn scan_rejects_symlinked_project_files() { + use std::os::unix::fs::symlink; + + let source = tempdir().unwrap(); + let outside = tempdir().unwrap(); + let target = outside.path().join("secret.md"); + fs::write(&target, project_markdown("Secret", "do not import")).unwrap(); + symlink(target, source.path().join("secret.md")).unwrap(); + + let error = scan_legacy_projects(source.path(), outside.path()).unwrap_err(); + assert!(error.contains("regular non-symlink")); + } + + #[test] + fn scan_rejects_oversize_and_invalid_utf8_files() { + let source = tempdir().unwrap(); + let backup = tempdir().unwrap(); + fs::write( + source.path().join("large.md"), + vec![b'a'; MAX_PROJECT_FILE_BYTES as usize + 1], + ) + .unwrap(); + assert!(scan_legacy_projects(source.path(), backup.path()) + .unwrap_err() + .contains("safety limit")); + + fs::remove_file(source.path().join("large.md")).unwrap(); + fs::write(source.path().join("invalid.md"), [0xff, 0xfe]).unwrap(); + assert!(scan_legacy_projects(source.path(), backup.path()) + .unwrap_err() + .contains("UTF-8")); + } + + #[tokio::test] + async fn legacy_import_cannot_run_after_native_activation() { + let (_dir, state) = state().await; + activate_for_test(&state).await; + + let error = apply_shadow_import( + state.pool(), + LegacyScan { + source_present: true, + source_root: Some("/legacy/projects".to_string()), + projects: Vec::new(), + }, + 0, + ) + .await + .unwrap_err(); + assert!(error.contains("disabled after native")); + } + + #[tokio::test] + async fn legacy_backfill_is_first_write_wins_and_uses_stable_backend_namespace() { + let (_db_dir, state) = state().await; + let source = tempdir().unwrap(); + let backup = tempdir().unwrap(); + fs::write( + source.path().join("alpha.md"), + project_markdown("Alpha", "Prompt"), + ) + .unwrap(); + apply_scan( + &state, + scan_legacy_projects(source.path(), backup.path()).unwrap(), + ) + .await + .unwrap(); + + let first = backfill_legacy_association( + state.pool(), + LegacySessionProjectBackfillRequest { + session_backend_id: "goose-serve".to_string(), + session_id: "session-1".to_string(), + project_id: "alpha".to_string(), + }, + ) + .await + .unwrap(); + assert_eq!(first.project_id.as_deref(), Some("alpha")); + assert_eq!(first.revision, Some(1)); + + let repeated = backfill_legacy_association( + state.pool(), + LegacySessionProjectBackfillRequest { + session_backend_id: "goose-serve".to_string(), + session_id: "session-1".to_string(), + project_id: "alpha".to_string(), + }, + ) + .await + .unwrap(); + assert_eq!(repeated, first); + + let other_backend = read_session_association(state.pool(), "external-acp", "session-1") + .await + .unwrap(); + assert!(!other_backend.has_association); + } + + #[tokio::test] + async fn native_association_uses_cas_and_preserves_explicit_null_tombstone() { + let (_dir, state) = state().await; + activate_for_test(&state).await; + + let tombstone = update_native_association( + state.pool(), + UpdateSessionProjectRequest { + session_backend_id: "goose-serve".to_string(), + session_id: "session-1".to_string(), + project_id: None, + expected_revision: 0, + }, + ) + .await + .unwrap(); + assert!(tombstone.has_association); + assert_eq!(tombstone.project_id, None); + assert_eq!(tombstone.revision, Some(1)); + + let conflict = update_native_association( + state.pool(), + UpdateSessionProjectRequest { + session_backend_id: "goose-serve".to_string(), + session_id: "session-1".to_string(), + project_id: None, + expected_revision: 0, + }, + ) + .await + .unwrap_err(); + assert!(conflict.contains("current revision is 1")); + + let regression = + sqlx::query("UPDATE project_storage_state SET phase = 'shadow' WHERE singleton_id = 1") + .execute(state.pool()) + .await + .unwrap_err(); + assert!(regression.to_string().contains("cannot move backwards")); + } + + #[tokio::test] + async fn concurrent_native_updates_serialize_into_one_success_and_one_cas_conflict() { + let (_dir, state) = state().await; + activate_for_test(&state).await; + update_native_association( + state.pool(), + UpdateSessionProjectRequest { + session_backend_id: "goose-serve".to_string(), + session_id: "session-1".to_string(), + project_id: None, + expected_revision: 0, + }, + ) + .await + .unwrap(); + + let first = update_native_association( + state.pool(), + UpdateSessionProjectRequest { + session_backend_id: "goose-serve".to_string(), + session_id: "session-1".to_string(), + project_id: None, + expected_revision: 1, + }, + ); + let second = update_native_association( + state.pool(), + UpdateSessionProjectRequest { + session_backend_id: "goose-serve".to_string(), + session_id: "session-1".to_string(), + project_id: None, + expected_revision: 1, + }, + ); + let (first, second) = tokio::join!(first, second); + assert_ne!(first.is_ok(), second.is_ok()); + let conflict = first.err().or_else(|| second.err()).unwrap(); + assert!(conflict.contains("current revision is 2")); + } + + #[tokio::test] + async fn stale_scan_generation_cannot_overwrite_a_newer_snapshot() { + let (_db_dir, state) = state().await; + let source = tempdir().unwrap(); + let backup = tempdir().unwrap(); + let project_path = source.path().join("alpha.md"); + + fs::write(&project_path, project_markdown("Alpha", "Old")).unwrap(); + let old_generation = claim_shadow_scan(state.pool()).await.unwrap(); + let old_scan = scan_legacy_projects(source.path(), backup.path()).unwrap(); + + fs::write(&project_path, project_markdown("Alpha", "New")).unwrap(); + let new_generation = claim_shadow_scan(state.pool()).await.unwrap(); + let new_scan = scan_legacy_projects(source.path(), backup.path()).unwrap(); + + apply_shadow_import(state.pool(), new_scan, new_generation) + .await + .unwrap(); + let stale = apply_shadow_import(state.pool(), old_scan, old_generation) + .await + .unwrap(); + assert!(stale.superseded); + assert_eq!( + sqlx::query_scalar::<_, String>("SELECT prompt FROM projects WHERE id = 'alpha'") + .fetch_one(state.pool()) + .await + .unwrap(), + "New" + ); + } + + #[tokio::test] + async fn unchanged_rescan_does_not_manufacture_a_collection_revision() { + let (_db_dir, state) = state().await; + let source = tempdir().unwrap(); + let backup = tempdir().unwrap(); + fs::write( + source.path().join("alpha.md"), + project_markdown("Alpha", "Prompt"), + ) + .unwrap(); + + apply_scan( + &state, + scan_legacy_projects(source.path(), backup.path()).unwrap(), + ) + .await + .unwrap(); + let before: (i64, Option) = sqlx::query_as( + "SELECT collection_revision, shadow_completed_at FROM project_storage_state", + ) + .fetch_one(state.pool()) + .await + .unwrap(); + + let report = apply_scan( + &state, + scan_legacy_projects(source.path(), backup.path()).unwrap(), + ) + .await + .unwrap(); + let after: (i64, Option) = sqlx::query_as( + "SELECT collection_revision, shadow_completed_at FROM project_storage_state", + ) + .fetch_one(state.pool()) + .await + .unwrap(); + assert_eq!(report.unchanged, 1); + assert_eq!(after, before); + let generations: (i64, i64) = sqlx::query_as( + "SELECT project_storage_state.shadow_scan_generation, project_migration_runs.scan_generation \ + FROM project_storage_state JOIN project_migration_runs ON migration_key = ? \ + WHERE singleton_id = 1", + ) + .bind(MIGRATION_KEY) + .fetch_one(state.pool()) + .await + .unwrap(); + assert_eq!(generations.0, generations.1); + } + + #[test] + fn workspace_import_matches_renderer_normalization_and_legacy_fallback() { + let explicit: BTreeMap = serde_json::from_value(serde_json::json!({ + "projectWorkspaces": [ + {"path": " C:\\\\Repo\\\\app\\\\.. ", "kind": "invalid", "source": "invalid", "startupMode": "branch", "repositoryPath": " C:\\\\Repo "}, + {"path": "c:/repo", "kind": "repository", "source": "selected", "startupMode": "worktree"} + ], + "workingDirs": ["/ignored"] + })) + .unwrap(); + let workspaces = parse_workspaces(&explicit).unwrap(); + assert_eq!(workspaces.len(), 1); + assert_eq!(workspaces[0].path, "c:/repo"); + assert_eq!(workspaces[0].kind, "repository"); + assert_eq!(workspaces[0].source, "selected"); + assert_eq!(workspaces[0].startup_mode, "auto-worktree"); + assert_eq!(workspaces[0].id, "path:c:/repo"); + + let fallback: BTreeMap = serde_json::from_value(serde_json::json!({ + "projectWorkspaces": [], + "workingDirs": [" /tmp/work ", ""], + "useWorktrees": true + })) + .unwrap(); + let workspaces = parse_workspaces(&fallback).unwrap(); + assert_eq!(workspaces.len(), 1); + assert_eq!(workspaces[0].path, "/tmp/work"); + assert_eq!(workspaces[0].startup_mode, "auto-worktree"); + + let duplicate_ids: BTreeMap = serde_json::from_value(serde_json::json!({ + "projectWorkspaces": [ + {"id": "explicit", "path": "/tmp/one"}, + {"id": "explicit", "path": "/tmp/two"} + ] + })) + .unwrap(); + let workspaces = parse_workspaces(&duplicate_ids).unwrap(); + assert_eq!(workspaces.len(), 2); + assert_eq!(workspaces[0].id, "explicit"); + assert_eq!(workspaces[1].id, "explicit"); + + assert_eq!(normalize_lexical_path("//server"), "/server"); + assert!(!is_windows_absolute_path("//server")); + assert!(is_windows_absolute_path("//server/share")); + assert_eq!(normalize_lexical_path("//server/./dir"), "//server/./dir"); + assert!(!is_windows_absolute_path("//server/./dir")); + assert_eq!(normalize_lexical_path("//../share"), "//../share"); + assert!(!is_windows_absolute_path("//../share")); + } + + #[tokio::test] + async fn import_accepts_existing_goose_project_filename_identity() { + let (_db_dir, state) = state().await; + let source = tempdir().unwrap(); + let backup = tempdir().unwrap(); + fs::write( + source.path().join("Legacy Project.md"), + project_markdown("Legacy Project", "Prompt"), + ) + .unwrap(); + + apply_scan( + &state, + scan_legacy_projects(source.path(), backup.path()).unwrap(), + ) + .await + .unwrap(); + assert_eq!( + sqlx::query_scalar::<_, String>("SELECT id FROM projects") + .fetch_one(state.pool()) + .await + .unwrap(), + "Legacy Project" + ); + } + + #[tokio::test] + async fn native_phase_allows_absent_legacy_backfill_but_existing_tombstone_wins() { + let (_db_dir, state) = state().await; + let source = tempdir().unwrap(); + let backup = tempdir().unwrap(); + fs::write( + source.path().join("alpha.md"), + project_markdown("Alpha", "Prompt"), + ) + .unwrap(); + apply_scan( + &state, + scan_legacy_projects(source.path(), backup.path()).unwrap(), + ) + .await + .unwrap(); + activate_for_test(&state).await; + + let backfilled = backfill_legacy_association( + state.pool(), + LegacySessionProjectBackfillRequest { + session_backend_id: "goose-serve".to_string(), + session_id: "session-new".to_string(), + project_id: "alpha".to_string(), + }, + ) + .await + .unwrap(); + assert_eq!(backfilled.project_id.as_deref(), Some("alpha")); + + update_native_association( + state.pool(), + UpdateSessionProjectRequest { + session_backend_id: "goose-serve".to_string(), + session_id: "session-cleared".to_string(), + project_id: None, + expected_revision: 0, + }, + ) + .await + .unwrap(); + let tombstone = backfill_legacy_association( + state.pool(), + LegacySessionProjectBackfillRequest { + session_backend_id: "goose-serve".to_string(), + session_id: "session-cleared".to_string(), + project_id: "missing-project".to_string(), + }, + ) + .await + .unwrap(); + assert!(tombstone.has_association); + assert_eq!(tombstone.project_id, None); + } + + #[tokio::test] + async fn project_delete_must_explicitly_revision_session_associations() { + let (_db_dir, state) = state().await; + let source = tempdir().unwrap(); + let backup = tempdir().unwrap(); + fs::write( + source.path().join("alpha.md"), + project_markdown("Alpha", "Prompt"), + ) + .unwrap(); + apply_scan( + &state, + scan_legacy_projects(source.path(), backup.path()).unwrap(), + ) + .await + .unwrap(); + activate_for_test(&state).await; + update_native_association( + state.pool(), + UpdateSessionProjectRequest { + session_backend_id: "goose-serve".to_string(), + session_id: "session-1".to_string(), + project_id: Some("alpha".to_string()), + expected_revision: 0, + }, + ) + .await + .unwrap(); + + let error = sqlx::query("DELETE FROM projects WHERE id = 'alpha'") + .execute(state.pool()) + .await + .unwrap_err(); + assert!(error.to_string().contains("FOREIGN KEY constraint failed")); + let association = read_session_association(state.pool(), "goose-serve", "session-1") + .await + .unwrap(); + assert_eq!(association.project_id.as_deref(), Some("alpha")); + assert_eq!(association.revision, Some(1)); + } + + #[tokio::test] + async fn newer_migration_version_is_never_downgraded_or_rewritten() { + let (_db_dir, state) = state().await; + sqlx::query( + "INSERT INTO project_migration_runs \ + (migration_key, version, state, checkpoint_json, completed_at, last_error, updated_at) \ + VALUES (?, 2, 'complete', '{\"future\":true}', 'later', NULL, 'later')", + ) + .bind(MIGRATION_KEY) + .execute(state.pool()) + .await + .unwrap(); + + assert!(claim_shadow_scan(state.pool()) + .await + .unwrap_err() + .contains("newer than this Berd build")); + assert!(record_import_failure(state.pool(), 0, "old build failure") + .await + .unwrap_err() + .contains("newer than this Berd build")); + let row: (i64, String, String) = sqlx::query_as( + "SELECT version, state, checkpoint_json FROM project_migration_runs WHERE migration_key = ?", + ) + .bind(MIGRATION_KEY) + .fetch_one(state.pool()) + .await + .unwrap(); + assert_eq!( + row, + (2, "complete".to_string(), "{\"future\":true}".to_string()) + ); + } + + #[tokio::test] + async fn accepted_retry_clears_failure_without_bumping_unchanged_collection() { + let (_db_dir, state) = state().await; + let source = tempdir().unwrap(); + let backup = tempdir().unwrap(); + fs::write( + source.path().join("alpha.md"), + project_markdown("Alpha", "Prompt"), + ) + .unwrap(); + apply_scan( + &state, + scan_legacy_projects(source.path(), backup.path()).unwrap(), + ) + .await + .unwrap(); + let collection_revision: i64 = sqlx::query_scalar( + "SELECT collection_revision FROM project_storage_state WHERE singleton_id = 1", + ) + .fetch_one(state.pool()) + .await + .unwrap(); + + let failed_generation = claim_shadow_scan(state.pool()).await.unwrap(); + assert!( + record_import_failure(state.pool(), failed_generation, "transient scan failure") + .await + .unwrap() + ); + let retry_generation = claim_shadow_scan(state.pool()).await.unwrap(); + let retry = scan_legacy_projects(source.path(), backup.path()).unwrap(); + apply_shadow_import(state.pool(), retry, retry_generation) + .await + .unwrap(); + + assert_eq!( + sqlx::query_scalar::<_, String>( + "SELECT state FROM project_migration_runs WHERE migration_key = ?", + ) + .bind(MIGRATION_KEY) + .fetch_one(state.pool()) + .await + .unwrap(), + "complete" + ); + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT collection_revision FROM project_storage_state WHERE singleton_id = 1", + ) + .fetch_one(state.pool()) + .await + .unwrap(), + collection_revision + ); + + let stale_generation = claim_shadow_scan(state.pool()).await.unwrap(); + let _newer_generation = claim_shadow_scan(state.pool()).await.unwrap(); + assert!( + !record_import_failure(state.pool(), stale_generation, "stale failure") + .await + .unwrap() + ); + assert_eq!( + sqlx::query_scalar::<_, String>( + "SELECT state FROM project_migration_runs WHERE migration_key = ?", + ) + .bind(MIGRATION_KEY) + .fetch_one(state.pool()) + .await + .unwrap(), + "complete" + ); + } + + #[cfg(unix)] + #[test] + fn scan_rejects_symlinked_backup_directory() { + use std::os::unix::fs::symlink; + + let source = tempdir().unwrap(); + let app_data = tempdir().unwrap(); + let outside = tempdir().unwrap(); + let backup_parent = app_data.path().join("project-migration-backups"); + fs::create_dir(&backup_parent).unwrap(); + symlink(outside.path(), backup_parent.join("v1")).unwrap(); + + let error = scan_legacy_projects(source.path(), &backup_parent.join("v1")).unwrap_err(); + assert!(error.contains("non-symlink directory")); + } + + #[test] + fn backup_quota_fails_before_writing_more_data() { + let backup = tempdir().unwrap(); + let mut total = MAX_BACKUP_BYTES; + let error = publish_backup(backup.path(), &"a".repeat(64), b"x", &mut total).unwrap_err(); + assert!(error.contains("safety limit")); + assert_eq!(fs::read_dir(backup.path()).unwrap().count(), 0); + } +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index a16f8664f..adae05e7a 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1,5 +1,6 @@ mod commands; mod deep_links; +mod persistence; mod services; mod types; @@ -247,15 +248,15 @@ pub fn run() { let release_channel_state = commands::updates::ReleaseChannelState::load(app.handle())?; app.manage(release_channel_state); - // `LayoutState::new` opens (and creates) the layout database, so the + // `BerdPersistenceState::new` opens (and creates) the shared database, so the // one-time legacy app-data migration must run first to copy any // pre-rename database before a fresh, empty one is created here. services::app_data_migration::migrate_legacy_app_data(app.handle()); - let layout_state = tauri::async_runtime::block_on(commands::layout::LayoutState::new( - app_data_dir.clone(), - )) + let persistence_state = tauri::async_runtime::block_on( + persistence::BerdPersistenceState::new(app_data_dir.clone()), + ) .map_err(std::io::Error::other)?; - app.manage(layout_state); + app.manage(persistence_state); // With all command state registered, it is now safe to run blocking, // async, network, or filesystem work. @@ -528,6 +529,11 @@ pub fn run() { commands::acp::get_goose_serve_host_info, commands::project_icons::scan_project_icons, commands::project_icons::read_project_icon, + commands::projects::get_project_storage_status, + commands::projects::shadow_import_legacy_projects, + commands::projects::backfill_legacy_session_project, + commands::projects::resolve_session_project, + commands::projects::update_native_session_project, commands::renderer::log_renderer_event, commands::artifacts::get_artifacts, commands::doctor::run_doctor, diff --git a/src-tauri/src/persistence.rs b/src-tauri/src/persistence.rs new file mode 100644 index 000000000..f7bc73793 --- /dev/null +++ b/src-tauri/src/persistence.rs @@ -0,0 +1,382 @@ +use chrono::Utc; +use serde::Serialize; +use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions, SqliteSynchronous}; +use sqlx::SqlitePool; +use std::fs::{self, File}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +pub(crate) const DATABASE_FILENAME: &str = "berd.sqlite"; +const RECOVERY_MARKER_SUFFIX: &str = ".recovery-required.json"; + +#[derive(Clone, Debug)] +pub struct BerdPersistenceState { + pub(crate) pool: SqlitePool, +} + +impl BerdPersistenceState { + pub async fn new(app_data_dir: PathBuf) -> Result { + fs::create_dir_all(&app_data_dir) + .map_err(|error| format!("Failed to create app data directory: {error}"))?; + Self::new_for_path(app_data_dir.join(DATABASE_FILENAME)).await + } + + #[cfg(test)] + pub(crate) async fn new_for_tests(db_path: PathBuf) -> Result { + Self::new_for_path(db_path).await + } + + async fn new_for_path(db_path: PathBuf) -> Result { + refuse_unresolved_recovery(&db_path)?; + if database_file_has_invalid_header(&db_path)? { + let marker = preserve_corrupt_database_files(&db_path)?; + return Err(recovery_required_error(&marker)); + } + + match open_migrated_pool(&db_path).await { + Ok(pool) => Ok(Self { pool }), + Err(error) if is_sqlite_corruption_error(&error) => { + let marker = preserve_corrupt_database_files(&db_path)?; + Err(recovery_required_error(&marker)) + } + Err(error) => Err(format!("Failed to initialize Berd database: {error}")), + } + } + + pub(crate) fn pool(&self) -> &SqlitePool { + &self.pool + } +} + +async fn open_migrated_pool(db_path: &Path) -> Result { + let options = SqliteConnectOptions::new() + .filename(db_path) + .create_if_missing(true) + .foreign_keys(true) + .journal_mode(SqliteJournalMode::Wal) + .synchronous(SqliteSynchronous::Full) + .busy_timeout(Duration::from_millis(5_000)); + let pool = SqlitePoolOptions::new() + .max_connections(5) + .connect_with(options) + .await?; + + if let Err(error) = sqlx::migrate!("./migrations").run(&pool).await { + pool.close().await; + return Err(error.into()); + } + if let Err(error) = read_initial_state(&pool).await { + pool.close().await; + return Err(error); + } + Ok(pool) +} + +async fn read_initial_state(pool: &SqlitePool) -> Result<(), sqlx::Error> { + sqlx::query("SELECT layout_id FROM layout_state WHERE layout_id = ?") + .bind(crate::commands::layout::HOME_LAYOUT_ID) + .fetch_one(pool) + .await?; + Ok(()) +} + +fn database_file_has_invalid_header(db_path: &Path) -> Result { + const SQLITE_HEADER: &[u8] = b"SQLite format 3\0"; + + if !db_path.exists() { + return Ok(false); + } + + let mut file = File::open(db_path) + .map_err(|error| format!("Failed to inspect Berd database header: {error}"))?; + let mut header = [0; SQLITE_HEADER.len()]; + let bytes_read = file + .read(&mut header) + .map_err(|error| format!("Failed to inspect Berd database header: {error}"))?; + Ok(bytes_read < SQLITE_HEADER.len() || header.as_slice() != SQLITE_HEADER) +} + +fn is_sqlite_corruption_error(error: &sqlx::Error) -> bool { + let mut current: Option<&(dyn std::error::Error + 'static)> = Some(error); + while let Some(error) = current { + if let Some(database_error) = error.downcast_ref::().and_then(|error| { + if let sqlx::Error::Database(database_error) = error { + Some(database_error) + } else { + None + } + }) { + if is_sqlite_corruption_code(database_error.code().as_deref()) { + return true; + } + } + current = error.source(); + } + false +} + +fn is_sqlite_corruption_code(code: Option<&str>) -> bool { + code.and_then(|code| code.parse::().ok()) + .is_some_and(|code| matches!(code & 0xff, 11 | 26)) +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct RecoveryMarker { + detected_at: String, + database_path: String, + preserved_files: Vec, +} + +fn recovery_marker_path(db_path: &Path) -> PathBuf { + PathBuf::from(format!("{}{}", db_path.display(), RECOVERY_MARKER_SUFFIX)) +} + +fn refuse_unresolved_recovery(db_path: &Path) -> Result<(), String> { + let marker = recovery_marker_path(db_path); + if marker.exists() { + return Err(recovery_required_error(&marker)); + } + Ok(()) +} + +fn recovery_required_error(marker: &Path) -> String { + format!( + "Berd database recovery is required. Preserved data and recovery details are recorded at {}. Refusing to create an empty replacement database.", + marker.display() + ) +} + +fn preserve_corrupt_database_files(db_path: &Path) -> Result { + let detected_at = Utc::now(); + let suffix = format!(".corrupt-{}", detected_at.format("%Y%m%dT%H%M%SZ")); + let files: Vec<(PathBuf, PathBuf)> = [ + db_path.to_path_buf(), + PathBuf::from(format!("{}-wal", db_path.display())), + PathBuf::from(format!("{}-shm", db_path.display())), + ] + .into_iter() + .filter(|path| path.exists()) + .map(|path| { + let file_name = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_else(|| DATABASE_FILENAME.to_string()); + let preserved = path.with_file_name(format!("{file_name}{suffix}")); + (path, preserved) + }) + .collect(); + + let marker_path = recovery_marker_path(db_path); + let marker = RecoveryMarker { + detected_at: detected_at.to_rfc3339(), + database_path: db_path.to_string_lossy().into_owned(), + preserved_files: files + .iter() + .map(|(_, preserved)| preserved.to_string_lossy().into_owned()) + .collect(), + }; + let marker_bytes = serde_json::to_vec_pretty(&marker) + .map_err(|error| format!("Failed to serialize database recovery marker: {error}"))?; + let temporary_marker = PathBuf::from(format!( + "{}.{}.tmp", + marker_path.display(), + uuid::Uuid::new_v4() + )); + let mut marker_file = File::create(&temporary_marker).map_err(|error| { + format!( + "Failed to write database recovery marker {}: {error}", + temporary_marker.display() + ) + })?; + marker_file.write_all(&marker_bytes).map_err(|error| { + format!( + "Failed to write database recovery marker {}: {error}", + temporary_marker.display() + ) + })?; + marker_file.sync_all().map_err(|error| { + format!( + "Failed to sync database recovery marker {}: {error}", + temporary_marker.display() + ) + })?; + drop(marker_file); + fs::rename(&temporary_marker, &marker_path).map_err(|error| { + format!( + "Failed to publish database recovery marker {}: {error}", + marker_path.display() + ) + })?; + sync_parent_directory(&marker_path)?; + + let mut errors = Vec::new(); + for (path, preserved) in &files { + match fs::rename(path, preserved) { + Ok(()) => log::error!( + "Preserved corrupt Berd database file {} as {}", + path.display(), + preserved.display() + ), + Err(error) => errors.push(format!( + "Failed to preserve corrupt database file {} as {}: {error}", + path.display(), + preserved.display() + )), + } + } + + if errors.is_empty() { + sync_parent_directory(db_path)?; + Ok(marker_path) + } else { + Err(format!( + "{}; recovery remains blocked by {}", + errors.join("; "), + marker_path.display() + )) + } +} + +#[cfg(unix)] +fn sync_parent_directory(path: &Path) -> Result<(), String> { + let parent = path + .parent() + .ok_or_else(|| format!("Database recovery path has no parent: {}", path.display()))?; + File::open(parent) + .and_then(|directory| directory.sync_all()) + .map_err(|error| { + format!( + "Failed to sync database recovery directory {}: {error}", + parent.display() + ) + }) +} + +#[cfg(not(unix))] +fn sync_parent_directory(_path: &Path) -> Result<(), String> { + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use sqlx::error::{DatabaseError, ErrorKind}; + use std::borrow::Cow; + use std::fmt; + use tempfile::tempdir; + + #[derive(Debug)] + struct FakeDatabaseError { + code: &'static str, + } + + impl fmt::Display for FakeDatabaseError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "fake sqlite error {}", self.code) + } + } + + impl std::error::Error for FakeDatabaseError {} + + impl DatabaseError for FakeDatabaseError { + fn message(&self) -> &str { + "fake sqlite error" + } + fn code(&self) -> Option> { + Some(Cow::Borrowed(self.code)) + } + fn as_error(&self) -> &(dyn std::error::Error + Send + Sync + 'static) { + self + } + fn as_error_mut(&mut self) -> &mut (dyn std::error::Error + Send + Sync + 'static) { + self + } + fn into_error(self: Box) -> Box { + self + } + fn kind(&self) -> ErrorKind { + ErrorKind::Other + } + } + + #[tokio::test] + async fn corruption_is_preserved_and_blocks_recreation_across_restarts() { + let dir = tempdir().expect("tempdir"); + let db_path = dir.path().join(DATABASE_FILENAME); + fs::write(&db_path, "not sqlite").expect("db"); + fs::write(format!("{}-wal", db_path.display()), "wal").expect("wal"); + fs::write(format!("{}-shm", db_path.display()), "shm").expect("shm"); + + let first_error = BerdPersistenceState::new_for_tests(db_path.clone()) + .await + .expect_err("corrupt database must fail closed"); + assert!(first_error.contains("recovery is required")); + assert!(!db_path.exists()); + + let second_error = BerdPersistenceState::new_for_tests(db_path.clone()) + .await + .expect_err("relaunch must remain blocked"); + assert!(second_error.contains("Refusing to create an empty replacement")); + + assert!(recovery_marker_path(&db_path).is_file()); + let renamed: Vec = fs::read_dir(dir.path()) + .expect("read dir") + .filter_map(Result::ok) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .filter(|name| name.contains(".corrupt-")) + .collect(); + assert_eq!(renamed.len(), 3); + } + + #[tokio::test] + async fn truncated_empty_database_is_corruption_and_cannot_be_recreated() { + let dir = tempdir().expect("tempdir"); + let db_path = dir.path().join(DATABASE_FILENAME); + fs::write(&db_path, []).expect("empty database"); + + let first_error = BerdPersistenceState::new_for_tests(db_path.clone()) + .await + .expect_err("an existing empty database must fail closed"); + assert!(first_error.contains("recovery is required")); + assert!(!db_path.exists()); + + let second_error = BerdPersistenceState::new_for_tests(db_path.clone()) + .await + .expect_err("relaunch must not migrate an empty replacement"); + assert!(second_error.contains("Refusing to create an empty replacement")); + assert!(recovery_marker_path(&db_path).is_file()); + } + + #[tokio::test] + async fn non_corruption_open_errors_do_not_create_recovery_marker() { + let dir = tempdir().expect("tempdir"); + let db_path = dir.path().join(DATABASE_FILENAME); + fs::create_dir(&db_path).expect("db path directory"); + + let error = BerdPersistenceState::new_for_tests(db_path.clone()) + .await + .expect_err("directory path should not be recoverable corruption"); + + assert!(error.contains("Failed to inspect Berd database header")); + assert!(db_path.is_dir()); + assert!(!recovery_marker_path(&db_path).exists()); + } + + #[test] + fn corruption_detector_matches_migration_errors_and_extended_codes() { + let top_level_corruption = sqlx::Error::database(FakeDatabaseError { code: "11" }); + assert!(is_sqlite_corruption_error(&top_level_corruption)); + let extended_corruption = sqlx::Error::database(FakeDatabaseError { code: "267" }); + assert!(is_sqlite_corruption_error(&extended_corruption)); + let migration_corruption = + sqlx::Error::Migrate(Box::new(sqlx::migrate::MigrateError::ExecuteMigration( + sqlx::Error::database(FakeDatabaseError { code: "267" }), + 20260519180000, + ))); + assert!(is_sqlite_corruption_error(&migration_corruption)); + let constraint_error = sqlx::Error::database(FakeDatabaseError { code: "2067" }); + assert!(!is_sqlite_corruption_error(&constraint_error)); + } +} diff --git a/src-tauri/src/services/e2e_mode.rs b/src-tauri/src/services/e2e_mode.rs index b9c5eee07..97ea57b99 100644 --- a/src-tauri/src/services/e2e_mode.rs +++ b/src-tauri/src/services/e2e_mode.rs @@ -174,6 +174,10 @@ impl E2eMode { self.goose_root.join(".agents") } + pub(crate) fn goose_data_dir(&self) -> PathBuf { + self.goose_root.join("data") + } + pub(crate) fn goose_agents_dir(&self) -> PathBuf { self.goose_agents_root().join("agents") } diff --git a/src-tauri/src/services/goose_config.rs b/src-tauri/src/services/goose_config.rs index 60a7f1b98..2974f99c7 100644 --- a/src-tauri/src/services/goose_config.rs +++ b/src-tauri/src/services/goose_config.rs @@ -7,7 +7,7 @@ use std::{ use etcetera::{choose_app_strategy, AppStrategy, AppStrategyArgs}; pub(crate) const ADDITIONAL_CONFIG_FILES_ENV: &str = "GOOSE_ADDITIONAL_CONFIG_FILES"; -const GOOSE_PATH_ROOT_ENV: &str = "GOOSE_PATH_ROOT"; +pub(crate) const GOOSE_PATH_ROOT_ENV: &str = "GOOSE_PATH_ROOT"; pub(crate) const CONFIG_FILE_NAME: &str = "config.yaml"; #[derive(Debug, Clone, PartialEq, Eq)] @@ -19,18 +19,40 @@ pub(crate) struct AdditionalConfigFiles { /// Resolve the upstream goose config file path. Matches /// `crates/goose/src/config/paths.rs::Paths::config_dir`. pub(crate) fn config_path() -> Result { - if let Some(root) = validated_path_root(env::var_os(GOOSE_PATH_ROOT_ENV)) { + config_path_from_values(env::var_os(GOOSE_PATH_ROOT_ENV), None) +} + +pub(crate) fn data_dir_from_values( + process_path_root: Option, + shell_path_root: Option, +) -> Result { + if let Some(root) = validated_path_root(shell_path_root.or(process_path_root)) { + return Ok(root.join("data")); + } + + let strategy = goose_app_strategy()?; + Ok(strategy.data_dir()) +} + +fn config_path_from_values( + process_path_root: Option, + shell_path_root: Option, +) -> Result { + if let Some(root) = validated_path_root(shell_path_root.or(process_path_root)) { return Ok(root.join("config").join(CONFIG_FILE_NAME)); } - let strategy = choose_app_strategy(AppStrategyArgs { + let strategy = goose_app_strategy()?; + Ok(strategy.config_dir().join(CONFIG_FILE_NAME)) +} + +fn goose_app_strategy() -> Result { + choose_app_strategy(AppStrategyArgs { top_level_domain: "Block".to_string(), author: "Block".to_string(), app_name: "goose".to_string(), }) - .map_err(|err| format!("Failed to resolve goose config directory: {err}"))?; - - Ok(strategy.config_dir().join(CONFIG_FILE_NAME)) + .map_err(|err| format!("Failed to resolve Goose application directory: {err}")) } fn validated_path_root(value: Option) -> Option { @@ -111,4 +133,19 @@ mod tests { Some(absolute) ); } + + #[test] + fn shell_path_root_matches_goose_serve_precedence() { + let process = std::env::current_dir().unwrap().join("process-root"); + let shell = std::env::current_dir().unwrap().join("shell-root"); + + assert_eq!( + data_dir_from_values( + Some(process.into_os_string()), + Some(shell.clone().into_os_string()), + ) + .unwrap(), + shell.join("data") + ); + } } diff --git a/src/features/projects/api/projects.shadow-import.test.ts b/src/features/projects/api/projects.shadow-import.test.ts new file mode 100644 index 000000000..dccd74043 --- /dev/null +++ b/src/features/projects/api/projects.shadow-import.test.ts @@ -0,0 +1,55 @@ +import { beforeEach, expect, it, vi } from "vitest"; +import type { ProjectInfo } from "./projects"; + +const mocks = vi.hoisted(() => ({ + invoke: vi.fn(), + listSources: vi.fn(), + deleteSource: vi.fn(), +})); + +vi.mock("@tauri-apps/api/core", () => ({ + invoke: mocks.invoke, +})); + +vi.mock("@/shared/api/acpConnection", () => ({ + getClient: vi.fn(async () => ({ + goose: { + GooseUnstableSourcesList: mocks.listSources, + GooseUnstableSourcesDelete: mocks.deleteSource, + }, + })), +})); + +beforeEach(() => { + vi.resetModules(); + mocks.invoke.mockReset(); + mocks.listSources.mockReset(); + mocks.deleteSource.mockReset(); + mocks.listSources.mockResolvedValue({ sources: [] }); + mocks.deleteSource.mockResolvedValue(undefined); +}); + +it("forces a rescan when a Goose mutation completes during a shadow scan", async () => { + let finishFirstScan: (() => void) | undefined; + mocks.invoke + .mockImplementationOnce( + () => + new Promise((resolve) => { + finishFirstScan = resolve; + }), + ) + .mockResolvedValue(undefined); + const { deleteProject, listProjects } = await import("./projects"); + + await listProjects(); + expect(mocks.invoke).toHaveBeenCalledTimes(1); + await deleteProject({ path: "/tmp/projects/launch.md" } as ProjectInfo); + expect(mocks.invoke).toHaveBeenCalledTimes(1); + + finishFirstScan?.(); + await vi.waitFor(() => expect(mocks.invoke).toHaveBeenCalledTimes(2)); + expect(mocks.invoke).toHaveBeenNthCalledWith( + 2, + "shadow_import_legacy_projects", + ); +}); diff --git a/src/features/projects/api/projects.ts b/src/features/projects/api/projects.ts index 34c87ecf3..1b4e18dcd 100644 --- a/src/features/projects/api/projects.ts +++ b/src/features/projects/api/projects.ts @@ -224,6 +224,38 @@ interface SourceEntry { properties: Record; } +let shadowImportPromise: Promise | null = null; +let shadowImportRequested = false; +let shadowImportLastStartedAt = 0; +const SHADOW_IMPORT_READ_INTERVAL_MS = 30_000; + +function startProjectStorageShadowImport(force = false): void { + const now = Date.now(); + if ( + !force && + (shadowImportPromise !== null || + now - shadowImportLastStartedAt < SHADOW_IMPORT_READ_INTERVAL_MS) + ) { + return; + } + shadowImportRequested = true; + if (shadowImportPromise) return; + shadowImportPromise = (async () => { + while (shadowImportRequested) { + shadowImportRequested = false; + shadowImportLastStartedAt = Date.now(); + try { + await invoke("shadow_import_legacy_projects"); + } catch (error) { + console.warn("Project storage shadow import failed", error); + } + } + })().finally(() => { + shadowImportPromise = null; + if (shadowImportRequested) startProjectStorageShadowImport(true); + }); +} + function toProjectInfo(source: SourceEntry): ProjectInfo { const p = source.properties ?? {}; const rawWorkingDirs = (p.workingDirs as string[]) ?? []; @@ -359,6 +391,7 @@ export async function listProjects(): Promise { const raw = await client.goose.GooseUnstableSourcesList({ type: "project", }); + startProjectStorageShadowImport(); const sources = (raw.sources ?? []) as unknown as SourceEntry[]; return sources .map(toProjectInfo) @@ -426,6 +459,7 @@ export async function createProject( chatGroups: null, }), }); + startProjectStorageShadowImport(true); return toProjectInfo(raw.source as SourceEntry); } @@ -498,6 +532,7 @@ export async function updateProject( chatGroups: merged.chatGroups, }), }); + startProjectStorageShadowImport(true); return toProjectInfo(raw.source as SourceEntry); } @@ -513,6 +548,7 @@ export async function deleteProject( type: "project", path, }); + startProjectStorageShadowImport(true); } export async function getProject(id: string): Promise { @@ -528,6 +564,7 @@ async function listAllProjects(): Promise { const raw = await client.goose.GooseUnstableSourcesList({ type: "project", }); + startProjectStorageShadowImport(); const sources = (raw.sources ?? []) as unknown as SourceEntry[]; return sources.map(toProjectInfo); }