diff --git a/CLAUDE.md b/CLAUDE.md index 29a608e1..f1a16c43 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -104,13 +104,15 @@ The hub also has a **Document Hub** database (design docs, specs) linked from ta Notion is where a task is born. Never create a Notion task from a GitHub issue — make it in Notion first, then sync it out to GitHub and record the issue URL in the task's `GitHub Link` property. The Notion UUID goes in the **PR body** (`Closes #N (Notion ID: )`), never in the issue description. -**One task means an entry in every tracker.** Creating the Notion page is half the job — fan it out and cross-link, without being asked: +**One task means an entry in both trackers.** Creating the Notion page is half the job — fan it out and cross-link, without being asked: 1. Notion page in the Tasks Tracker. 2. `gh issue create --title "ASNT-: "`, body ending `**Notion:** `. Match the structure existing issues use (Overview / Scope / Task Details / Definition of Done / Estimate / Type). 3. Write the issue URL back into the Notion `GitHub Link` property. -Linear is no longer used — don't create Linear issues or fill the `Linear Link` property. +**Linear is not used.** Never create a Linear issue, update the status of an existing one, or fill +the Notion `Linear Link` property — that property and the Linear issues behind it are leftovers. +Sub-task nesting and `Blocked by`/`Blocking` live in Notion. ## Code Style — no bloat diff --git a/Cargo.lock b/Cargo.lock index 65b639bb..5803f876 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1751,6 +1751,7 @@ dependencies = [ "base64", "calamine 0.26.1", "csv", + "file-ingest", "getrandom 0.3.4", "log", "quick-xml 0.37.5", @@ -2341,6 +2342,13 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "file-ingest" +version = "0.5.0-beta.1" +dependencies = [ + "tempfile", +] + [[package]] name = "filedescriptor" version = "0.8.3" @@ -6387,6 +6395,7 @@ dependencies = [ "data-exchange", "diagnostics", "dirs 6.0.0", + "file-ingest", "gpui-component", "gpui-pre", "log", @@ -6394,6 +6403,7 @@ dependencies = [ "serde", "serde_json", "settings", + "tempfile", "window-wrapper", ] @@ -8401,6 +8411,7 @@ dependencies = [ "base64", "clustering", "diagnostics", + "file-ingest", "gpui-base", "gpui-component", "gpui-pre", diff --git a/Cargo.toml b/Cargo.toml index 7e89af38..292896fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/clustering", "crates/data-exchange", "crates/diagnostics", + "crates/file-ingest", "crates/plugin-api", "crates/plugin-host", "crates/plugin-package", diff --git a/crates/app/src/actions.rs b/crates/app/src/actions.rs index aa5d4764..a95d8b0f 100644 --- a/crates/app/src/actions.rs +++ b/crates/app/src/actions.rs @@ -84,6 +84,18 @@ pub fn key_bindings() -> Vec { KeyBinding::new("secondary-x", table::Cut, Some(table::GRID_CONTEXT)), KeyBinding::new("secondary-c", table::Copy, Some(table::GRID_CONTEXT)), KeyBinding::new("secondary-v", table::Paste, Some(table::GRID_CONTEXT)), + KeyBinding::new( + "secondary-alt-right", + table::ExpandAll, + Some(table::GRID_CONTEXT), + ), + KeyBinding::new( + "secondary-alt-left", + table::CollapseAll, + Some(table::GRID_CONTEXT), + ), + KeyBinding::new("secondary-]", table::IndentRow, Some(table::GRID_CONTEXT)), + KeyBinding::new("secondary-[", table::OutdentRow, Some(table::GRID_CONTEXT)), // Blank the selection, the spreadsheet convention — and both keys, since Sheets and Excel // accept either. Scoped to the grid like the clipboard keys above, which is what keeps // Backspace deleting *text* while the cell editor or the find bar holds focus. diff --git a/crates/app/src/app_menus.rs b/crates/app/src/app_menus.rs index 62b18735..fd008b75 100644 --- a/crates/app/src/app_menus.rs +++ b/crates/app/src/app_menus.rs @@ -20,6 +20,8 @@ actions!( /// The same window as `OpenSettings`, opened on the Columns page — the Data menu promises /// a column surface, and landing on whatever page was last used does not keep that promise. OpenColumnSettings, + /// Re-type the open project's columns from a column config file or Sheet. + LoadColumnConfig, DiscoverPlugins, ManagePlugins, OpenPluginsFolder, @@ -104,6 +106,9 @@ fn app_menus(cx: &gpui::App) -> Vec { MenuItem::Separator, MenuItem::action("Open Projects…", OpenProjects), MenuItem::action("Save", Save), + MenuItem::Separator, + MenuItem::action("Import Files or Folders…", table::ImportFiles), + MenuItem::action("Relink Missing Files…", table::RelinkMissingFiles), MenuItem::submenu(Menu { name: "Export".into(), disabled: false, @@ -193,6 +198,7 @@ fn app_menus(cx: &gpui::App) -> Vec { disabled: false, items: vec![ MenuItem::action("Column Settings…", OpenColumnSettings), + MenuItem::action("Load Column Config…", LoadColumnConfig), MenuItem::action("Rename Column…", RenameColumn), MenuItem::Separator, MenuItem::action("Delete Row", DeleteRow), diff --git a/crates/app/src/export.rs b/crates/app/src/export.rs index d120bfc7..d0c21bd8 100644 --- a/crates/app/src/export.rs +++ b/crates/app/src/export.rs @@ -9,7 +9,7 @@ use std::io::Write as _; use std::path::{Path, PathBuf}; use std::rc::Rc; -use data_exchange::export::{self, CSL_FIELDS, CslMapping}; +use data_exchange::export::{self, ArchiveFile, CSL_FIELDS, CslMapping}; use gpui::{ Action, App, AppContext as _, ClickEvent, IntoElement, ParentElement, SharedString, Styled, Window, @@ -26,6 +26,13 @@ use settings::project::CurrentProject; /// Where the CSL picker's answer is remembered, per project. const CSL_MAPPING_KEY: &str = "csl_mapping"; +struct ExportGrid { + headers: Vec, + row_ids: Vec, + rows: Vec>, + structure: Vec, +} + #[derive(Clone, Copy, PartialEq, Eq, Deserialize, JsonSchema)] pub enum ExportFormat { Csv, @@ -202,12 +209,41 @@ pub fn run(format: ExportFormat, window: &mut Window, cx: &mut App) { }); return; } - let (Some(project), Some((headers, rows))) = (cx.try_global::(), grid(cx)) + let (Some(project), Some((headers, mut rows))) = (cx.try_global::(), grid(cx)) else { log::warn!("export was asked for with no project open"); return; }; let (file, title) = (project.file.clone(), project.display_name()); + let (row_ids, structure) = cx + .try_global::() + .and_then(|handle| handle.0.upgrade()) + .map(|state| { + let state = state.read(cx); + let (_, row_ids, _) = state.delegate().dataset_snapshot(); + (row_ids, state.delegate().row_structure().to_vec()) + }) + .unwrap_or_default(); + let declared: Vec<_> = project + .data + .columns + .iter() + .map(|column| { + ( + column.name.clone(), + ColumnType::from_declared(&column.data_type), + ) + }) + .collect(); + let description = settings::description::DescriptionConfig::from_values(&project.data.values); + export::project_structure_columns( + &headers, + &row_ids, + &mut rows, + &structure, + &declared, + &description, + ); if is_google(format) { // A project that already knows its spreadsheet refills that one; otherwise "Sync" asks @@ -240,13 +276,35 @@ pub fn run(format: ExportFormat, window: &mut Window, cx: &mut App) { .unwrap_or_default(); table::photos::resolve_row_images(&headers, &rows, &folder, &declared) .into_iter() - .flatten() + .enumerate() + .filter_map(|(index, path)| { + let path = path?; + let source_path = row_ids.get(index).and_then(|row_id| { + structure + .iter() + .find(|component| component.row_id == *row_id) + .and_then(|component| component.source_path.clone()) + }); + Some(ArchiveFile { path, source_path }) + }) .collect() } else { Vec::new() }; - save_as(format, file, headers, rows, images, CslMapping::new(), cx); + save_as( + format, + file, + ExportGrid { + headers, + row_ids, + rows, + structure, + }, + images, + CslMapping::new(), + cx, + ); } /// Ask where it goes, then write it off the UI thread. The ZIP copies image bytes and the others @@ -254,12 +312,17 @@ pub fn run(format: ExportFormat, window: &mut Window, cx: &mut App) { fn save_as( format: ExportFormat, project_file: PathBuf, - headers: Vec, - rows: Vec>, - images: Vec, + grid: ExportGrid, + images: Vec, mapping: CslMapping, cx: &mut App, ) { + let ExportGrid { + headers, + row_ids, + rows, + structure, + } = grid; let directory = project_file .parent() .map_or_else(|| PathBuf::from("."), Path::to_path_buf); @@ -275,13 +338,16 @@ fn save_as( }; let result = match format { ExportFormat::Csv => export::write_csv(&path, &headers, &rows), - ExportFormat::JsonLd => { - export::write_json(&path, &export::jsonld_value(&headers, &rows)) - } + ExportFormat::JsonLd => export::write_json( + &path, + &export::jsonld_hierarchy_value(&headers, &row_ids, &rows, &structure), + ), ExportFormat::Csl => { export::write_json(&path, &export::csl_items(&headers, &rows, &mapping)) } - ExportFormat::Zip => export::write_zip(&path, &headers, &rows, &images), + ExportFormat::Zip => { + export::write_zip(&path, &headers, &row_ids, &rows, &structure, &images) + } // Handled in `run` — they have no path to write to. ExportFormat::GoogleSheet | ExportFormat::GoogleSheetSync => return, }; @@ -393,8 +459,12 @@ fn ask_csl_mapping( save_as( ExportFormat::Csl, project_file.clone(), - headers.clone(), - rows.clone(), + ExportGrid { + headers: headers.clone(), + row_ids: Vec::new(), + rows: rows.clone(), + structure: Vec::new(), + }, Vec::new(), mapping, cx, diff --git a/crates/app/src/main.rs b/crates/app/src/main.rs index baa56b2b..54d8b49d 100644 --- a/crates/app/src/main.rs +++ b/crates/app/src/main.rs @@ -42,8 +42,9 @@ use crate::app_settings::build_pages; use crate::{ actions::{NewProject, ToggleBottomDock, ToggleLeftDock, ToggleRightDock}, app_menus::{ - CopyDebugInfo, OpenAbout, OpenColumnSettings, OpenLogsFolder, OpenPluginsFolder, - OpenProjects, OpenSettings, Quit, ReloadPlugins, ReportBug, ReportUxIssue, RequestFeature, + CopyDebugInfo, LoadColumnConfig, OpenAbout, OpenColumnSettings, OpenLogsFolder, + OpenPluginsFolder, OpenProjects, OpenSettings, Quit, ReloadPlugins, ReportBug, + ReportUxIssue, RequestFeature, }, status_items::build_status_bar_registry, title_items::build_title_bar_registry, @@ -291,6 +292,10 @@ impl Render for App { .on_action(cx.listener(|_, action: &export::Export, window, cx| { export::run(action.format, window, cx) })) + // Here for the same reason: the dialog opens in this window. + .on_action(cx.listener(|_, _: &LoadColumnConfig, window, cx| { + project_wizard::open_column_config_dialog(window, cx) + })) .on_action( cx.listener(|_, action: &export::PluginExport, _, cx| { export::run_plugin(action, cx) diff --git a/crates/app/src/status_items/cell_location.rs b/crates/app/src/status_items/cell_location.rs index fff760f5..d2cd611e 100644 --- a/crates/app/src/status_items/cell_location.rs +++ b/crates/app/src/status_items/cell_location.rs @@ -1,5 +1,5 @@ -//! Right-side status-bar widget: shows the table's selection — `Row N, Col M, K words` for a -//! cell (word count of its text), `Row N` for a whole-row selection, `Col M` for a whole-column +//! Right-side status-bar widget: shows the table's selection — `Row N, Col M, K characters` for a +//! cell (character count of its text), `Row N` for a whole-row selection, `Col M` for a whole-column //! selection. Text-only readout — sits leftmost in the right group. use gpui::*; @@ -52,12 +52,18 @@ impl CellLocation { } match state.delegate().selection() { Some(Selection::Cell { row, col }) => { - let words = state + let characters = state .delegate() .cell(row, col) - .map(|t| t.split_whitespace().count()) + .map(|t| t.chars().count()) .unwrap_or(0); - format!("Row {}, Col {}, {} words", row + 1, col + 1, words).into() + format!( + "Row {}, Col {}, {} characters", + row + 1, + col + 1, + characters + ) + .into() } Some(Selection::Row(row)) => format!("Row {}", row + 1).into(), Some(Selection::Column(col)) => format!("Col {}", col + 1).into(), diff --git a/crates/data-exchange/Cargo.toml b/crates/data-exchange/Cargo.toml index 420d78ed..6f2a5135 100644 --- a/crates/data-exchange/Cargo.toml +++ b/crates/data-exchange/Cargo.toml @@ -9,6 +9,7 @@ csv.workspace = true serde.workspace = true serde_json.workspace = true settings = { path = "../settings" } +file-ingest = { path = "../file-ingest" } # blocking, not async: gpui's executor isn't a tokio reactor, so we run the # one-shot fetch on the background executor with the blocking client. reqwest = { version = "0.12", features = ["blocking", "json"] } diff --git a/crates/data-exchange/src/export.rs b/crates/data-exchange/src/export.rs index cdb71601..0df0ed3f 100644 --- a/crates/data-exchange/src/export.rs +++ b/crates/data-exchange/src/export.rs @@ -24,6 +24,12 @@ pub enum ExportError { Zip(#[from] zip::result::ZipError), } +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ArchiveFile { + pub path: PathBuf, + pub source_path: Option, +} + /// The grid as CSV: the header row, then every row in table order. fn csv_bytes(headers: &[String], rows: &[Vec]) -> Result, csv::Error> { let mut writer = csv::Writer::from_writer(Vec::new()); @@ -43,19 +49,104 @@ pub fn write_json(path: &Path, value: &Value) -> Result<(), ExportError> { Ok(serde_json::to_writer_pretty(File::create(path)?, value).map_err(std::io::Error::from)?) } +/// Fills user-declared archival projection columns without adding columns to the dataset. +pub fn project_structure_columns( + headers: &[String], + row_ids: &[settings::project::RowId], + rows: &mut [Vec], + structure: &[settings::project::RowStructure], + columns: &[(String, ColumnType)], + description: &settings::description::DescriptionConfig, +) { + let by_id: std::collections::HashMap<_, _> = structure + .iter() + .map(|component| (component.row_id, component)) + .collect(); + let title_col = columns + .iter() + .find(|(_, kind)| *kind == ColumnType::Title) + .and_then(|(name, _)| headers.iter().position(|header| header == name)); + let titles: std::collections::HashMap<_, _> = title_col + .map(|title| { + row_ids + .iter() + .zip(rows.iter()) + .filter_map(|(id, row)| row.get(title).map(|value| (*id, value.clone()))) + .collect() + }) + .unwrap_or_default(); + let level_labels: std::collections::HashMap<_, _> = description + .levels + .iter() + .map(|level| (level.key.as_str(), level.label.as_str())) + .collect(); + for (source, row) in rows.iter_mut().enumerate() { + let Some(component) = row_ids.get(source).and_then(|id| by_id.get(id)) else { + continue; + }; + for (name, kind) in columns { + let Some(col) = headers.iter().position(|header| header == name) else { + continue; + }; + // A value the archivist typed stays theirs; only a blank cell is filled from structure. + if row.get(col).is_none_or(|value| !value.trim().is_empty()) { + continue; + } + row[col] = match kind { + ColumnType::DescriptionLevel => level_labels + .get(component.level_key.as_str()) + .copied() + .unwrap_or(component.level_key.as_str()) + .to_string(), + ColumnType::ParentComponent => component + .parent_id + .map(|parent_id| { + titles + .get(&parent_id) + .filter(|title| !title.trim().is_empty()) + .cloned() + .unwrap_or_else(|| parent_id.to_string()) + }) + .unwrap_or_default(), + ColumnType::SourcePath => component.source_path.clone().unwrap_or_default(), + _ => continue, + }; + } + } +} + /// One node per row, keyed by the column headers verbatim. The headers are the vocabulary a /// collection already uses, so `@vocab` resolves them rather than a mapping table nobody wrote — -/// a reader gets terms that match the spreadsheet they came from. -pub fn jsonld_value(headers: &[String], rows: &[Vec]) -> Value { +/// a reader gets terms that match the spreadsheet they came from. Each node also carries a stable +/// component `@id` and, when it has a parent, an `isPartOf` whole-part link. +pub fn jsonld_hierarchy_value( + headers: &[String], + row_ids: &[settings::project::RowId], + rows: &[Vec], + structure: &[settings::project::RowStructure], +) -> Value { + let structure: std::collections::HashMap<_, _> = + structure.iter().map(|item| (item.row_id, item)).collect(); let graph: Vec = rows .iter() - .map(|row| { - let node: Map = headers + .zip(row_ids) + .map(|(row, row_id)| { + let mut node: Map = headers .iter() .zip(row) .filter(|(_, cell)| !cell.trim().is_empty()) .map(|(header, cell)| (header.clone(), Value::String(cell.clone()))) .collect(); + node.insert("@id".into(), format!("urn:qrate:component:{row_id}").into()); + if let Some(component) = structure.get(row_id) { + node.insert("additionalType".into(), component.level_key.clone().into()); + if let Some(parent_id) = component.parent_id { + node.insert( + "isPartOf".into(), + json!({ "@id": format!("urn:qrate:component:{parent_id}") }), + ); + } + } Value::Object(node) }) .collect(); @@ -156,8 +247,10 @@ pub fn csl_items(headers: &[String], rows: &[Vec], mapping: &CslMapping) pub fn write_zip( path: &Path, headers: &[String], + row_ids: &[settings::project::RowId], rows: &[Vec], - images: &[PathBuf], + structure: &[settings::project::RowStructure], + images: &[ArchiveFile], ) -> Result<(), ExportError> { let mut zip = zip::ZipWriter::new(File::create(path)?); let text = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); @@ -168,41 +261,70 @@ pub fn write_zip( zip.write_all(&csv_bytes(headers, rows)?)?; zip.start_file("metadata.jsonld", text)?; zip.write_all( - &serde_json::to_vec_pretty(&jsonld_value(headers, rows)).map_err(std::io::Error::from)?, + &serde_json::to_vec_pretty(&jsonld_hierarchy_value(headers, row_ids, rows, structure)) + .map_err(std::io::Error::from)?, )?; let mut taken: HashSet = HashSet::new(); for image in images { - let Some(name) = image.file_name().and_then(|n| n.to_str()) else { + let Some(fallback) = image.path.file_name().and_then(|n| n.to_str()) else { continue; }; + let relative = image + .source_path + .as_deref() + .and_then(safe_archive_path) + .unwrap_or_else(|| fallback.to_string()); // Two folders can hold the same filename, and a zip entry that repeats one silently wins. - let mut name = name.to_string(); + let mut name = relative; for n in 2.. { if taken.insert(name.clone()) { break; } let stem = Path::new(&name).file_stem().unwrap_or_default(); - name = format!("{}_{n}", stem.to_string_lossy()); - if let Some(ext) = image.extension().and_then(|e| e.to_str()) { - name = format!("{name}.{ext}"); + let parent = Path::new(&name).parent().unwrap_or_else(|| Path::new("")); + let mut renamed = format!("{}_{n}", stem.to_string_lossy()); + if let Some(ext) = image.path.extension().and_then(|e| e.to_str()) { + renamed = format!("{renamed}.{ext}"); } + name = file_ingest::normalized_path(&parent.join(renamed)); } - match std::fs::read(image) { + match std::fs::read(&image.path) { Ok(bytes) => { zip.start_file(format!("files/{name}"), binary)?; zip.write_all(&bytes)?; } - Err(err) => log::warn!("left {} out of the export archive: {err}", image.display()), + Err(err) => log::warn!( + "left {} out of the export archive: {err}", + image.path.display() + ), } } zip.finish()?; Ok(()) } +fn safe_archive_path(source: &str) -> Option { + let path = Path::new(source); + if path.is_absolute() || source.trim().is_empty() { + return None; + } + let safe: PathBuf = path + .components() + .filter_map(|component| match component { + std::path::Component::Normal(part) => Some(part), + _ => None, + }) + .collect(); + (!safe.as_os_str().is_empty()).then(|| file_ingest::normalized_path(&safe)) +} + #[cfg(test)] mod tests { - use super::{CslMapping, csl_items, derive_csl_mapping, jsonld_value, write_zip}; + use super::{ + ArchiveFile, CslMapping, csl_items, derive_csl_mapping, jsonld_hierarchy_value, + project_structure_columns, write_zip, + }; use settings::columns::ColumnType; fn grid() -> (Vec, Vec>) { @@ -221,15 +343,105 @@ mod tests { #[test] fn jsonld_keeps_column_order_and_drops_empty_cells() { let (headers, rows) = grid(); - let doc = jsonld_value(&headers, &rows); + let doc = jsonld_hierarchy_value(&headers, &[1, 2], &rows, &[]); let graph = doc["@graph"].as_array().unwrap(); assert_eq!(graph.len(), 2); assert_eq!( graph[0].as_object().unwrap().keys().collect::>(), - ["Digital ID", "Title", "Taken", "Notes"] + ["Digital ID", "Title", "Taken", "Notes", "@id"] ); // The second row's blanks are absent, not empty strings — a null value is a claim. - assert_eq!(graph[1].as_object().unwrap().len(), 2); + assert_eq!(graph[1].as_object().unwrap().len(), 3); + } + + #[test] + fn jsonld_exports_archival_whole_part_relationships() { + let (headers, rows) = grid(); + let structure = [ + settings::project::RowStructure { + row_id: 10, + parent_id: None, + level_key: "series".into(), + sibling_order: 0, + source_path: None, + source_kind: None, + }, + settings::project::RowStructure { + row_id: 11, + parent_id: Some(10), + level_key: "item".into(), + sibling_order: 0, + source_path: None, + source_kind: None, + }, + ]; + let graph = jsonld_hierarchy_value(&headers, &[10, 11], &rows, &structure); + assert_eq!(graph["@graph"][0]["additionalType"], "series"); + assert_eq!( + graph["@graph"][1]["isPartOf"]["@id"], + "urn:qrate:component:10" + ); + } + + #[test] + fn declared_structure_columns_are_filled_without_adding_columns() { + let headers = vec![ + "Title".into(), + "Level".into(), + "Parent".into(), + "Path".into(), + ]; + let mut rows = vec![ + vec![ + "Photographs".into(), + "Fonds".into(), + String::new(), + String::new(), + ], + vec![ + "one.jpg".into(), + String::new(), + String::new(), + String::new(), + ], + ]; + let structure = [ + settings::project::RowStructure { + row_id: 10, + parent_id: None, + level_key: "series".into(), + sibling_order: 0, + source_path: Some("Photographs".into()), + source_kind: None, + }, + settings::project::RowStructure { + row_id: 11, + parent_id: Some(10), + level_key: "item".into(), + sibling_order: 0, + source_path: Some("Photographs/one.jpg".into()), + source_kind: None, + }, + ]; + project_structure_columns( + &headers, + &[10, 11], + &mut rows, + &structure, + &[ + ("Title".into(), ColumnType::Title), + ("Level".into(), ColumnType::DescriptionLevel), + ("Parent".into(), ColumnType::ParentComponent), + ("Path".into(), ColumnType::SourcePath), + ], + &settings::description::DescriptionProfile::Rad.defaults(), + ); + assert_eq!( + rows[1], + ["one.jpg", "Item", "Photographs", "Photographs/one.jpg"] + ); + assert_eq!(rows[0][1], "Fonds", "a typed value is never overwritten"); + assert_eq!(headers.len(), 4); } #[test] @@ -275,20 +487,31 @@ mod tests { write_zip( &archive, &headers, + &[10, 11], &rows, - &[dir.join("a/1.jpg"), dir.join("b/1.jpg")], + &[], + &[ + ArchiveFile { + path: dir.join("a/1.jpg"), + source_path: Some("a/1.jpg".into()), + }, + ArchiveFile { + path: dir.join("b/1.jpg"), + source_path: Some("b/1.jpg".into()), + }, + ], ) .unwrap(); let zip = zip::ZipArchive::new(std::fs::File::open(&archive).unwrap()).unwrap(); - // Two files sharing a name both survive; the second is renamed rather than overwriting. + // Two files sharing a name both survive in their source directories. assert_eq!( zip.file_names().collect::>(), [ "data.csv", "metadata.jsonld", - "files/1.jpg", - "files/1_2.jpg" + "files/a/1.jpg", + "files/b/1.jpg" ] .into_iter() .collect() diff --git a/crates/file-ingest/Cargo.toml b/crates/file-ingest/Cargo.toml new file mode 100644 index 00000000..637cef44 --- /dev/null +++ b/crates/file-ingest/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "file-ingest" +version.workspace = true +edition.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/crates/file-ingest/src/duplicates.rs b/crates/file-ingest/src/duplicates.rs new file mode 100644 index 00000000..3c8ba7b0 --- /dev/null +++ b/crates/file-ingest/src/duplicates.rs @@ -0,0 +1,369 @@ +//! What an import does about material the project already holds. +//! +//! Digitization arrives in batches, so the same folder is dropped again with new files in it. This +//! module decides, without touching a project, which planned components are already components. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use crate::{EntryKind, ImportPlan, normalized_path}; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum DuplicatePolicy { + /// Leave what is already catalogued alone. Re-importing a batch folder is then a no-op. + #[default] + Skip, + /// Keep the existing row and its typed metadata, re-linking it to the file just imported. + Update, + AddAsNew, +} + +impl DuplicatePolicy { + pub fn key(self) -> &'static str { + match self { + Self::Skip => "skip", + Self::Update => "update", + Self::AddAsNew => "add_as_new", + } + } + + pub fn parse(value: &str) -> Self { + match value.trim() { + "update" => Self::Update, + "add_as_new" => Self::AddAsNew, + _ => Self::Skip, + } + } +} + +/// One component the project already holds. `key` is the caller's own identifier — a row id in the +/// open table, a row index in the wizard — and comes back untouched in a [`Resolution`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExistingComponent { + pub key: u64, + /// Absolute path of the file this component is linked to, when it has one. + pub absolute_source: Option, + /// `settings::filenames::lookup_keys` of whatever names its file. + pub filename_keys: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Resolution { + Create, + Skip { + existing: u64, + }, + Update { + existing: u64, + }, + /// Several existing components could be this file. Never merged automatically. + Ambiguous { + candidates: Vec, + }, +} + +/// Where a component belongs once duplicates are resolved: under another planned component, or +/// under one the project already holds, so a second batch lands inside the first batch's series. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Parent { + Planned(usize), + Existing(u64), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ResolvedPlan { + pub plan: ImportPlan, + /// Parallel to `plan.components`. + pub resolutions: Vec, + /// Parallel to `plan.components`. + pub parents: Vec>, +} + +impl ResolvedPlan { + pub fn created(&self) -> usize { + self.count(|resolution| matches!(resolution, Resolution::Create)) + } + + pub fn skipped(&self) -> usize { + self.count(|resolution| matches!(resolution, Resolution::Skip { .. })) + } + + pub fn updated(&self) -> usize { + self.count(|resolution| matches!(resolution, Resolution::Update { .. })) + } + + pub fn ambiguous(&self) -> usize { + self.count(|resolution| matches!(resolution, Resolution::Ambiguous { .. })) + } + + /// Components the policy did not leave as new material. + pub fn duplicates(&self) -> usize { + self.skipped() + self.updated() + } + + fn count(&self, matching: impl Fn(&Resolution) -> bool) -> usize { + self.resolutions + .iter() + .filter(|resolution| matching(resolution)) + .count() + } +} + +/// Matches `plan` against what the project already holds. +/// +/// `file_keys` is the host's filename-key rule (`settings::filenames::keys`), passed in so this +/// crate stays free of project types and both callers match the same way. +pub fn resolve( + plan: ImportPlan, + existing: &[ExistingComponent], + file_keys: impl Fn(&str) -> Vec, + policy: DuplicatePolicy, +) -> ResolvedPlan { + let plan = collapse_repeats(plan); + let by_source: HashMap = existing + .iter() + .filter_map(|component| { + let source = component.absolute_source.as_deref()?; + Some((comparable(source), component.key)) + }) + .collect(); + let mut by_key: HashMap<&str, Vec> = HashMap::new(); + for component in existing { + for key in &component.filename_keys { + by_key.entry(key.as_str()).or_default().push(component.key); + } + } + + let resolutions: Vec = plan + .components + .iter() + .map(|component| { + if let Some(existing) = by_source.get(&comparable(&component.absolute_path)) { + return matched(policy, *existing); + } + if component.kind == EntryKind::Directory { + return Resolution::Create; + } + let mut candidates: Vec = file_keys(&normalized_path(&component.absolute_path)) + .iter() + .filter_map(|key| by_key.get(key.as_str())) + .flatten() + .copied() + .collect(); + candidates.sort_unstable(); + candidates.dedup(); + match candidates.len() { + 0 => Resolution::Create, + 1 => matched(policy, candidates[0]), + _ => Resolution::Ambiguous { candidates }, + } + }) + .collect(); + + let parents = plan + .components + .iter() + .map(|component| { + let parent = component.parent?; + Some(match &resolutions[parent] { + Resolution::Skip { existing } | Resolution::Update { existing } => { + Parent::Existing(*existing) + } + _ => Parent::Planned(parent), + }) + }) + .collect(); + + ResolvedPlan { + plan, + resolutions, + parents, + } +} + +fn matched(policy: DuplicatePolicy, existing: u64) -> Resolution { + match policy { + DuplicatePolicy::Skip => Resolution::Skip { existing }, + DuplicatePolicy::Update => Resolution::Update { existing }, + DuplicatePolicy::AddAsNew => Resolution::Create, + } +} + +/// One path reached twice in a single import — a folder dropped alongside a file inside it — is +/// one component, not a question for the archivist. +fn collapse_repeats(plan: ImportPlan) -> ImportPlan { + let mut kept_at: HashMap = HashMap::new(); + let mut moved_to: Vec = Vec::with_capacity(plan.components.len()); + let mut components = Vec::with_capacity(plan.components.len()); + for component in plan.components { + match kept_at.get(&comparable(&component.absolute_path)) { + Some(kept) => moved_to.push(*kept), + None => { + kept_at.insert(comparable(&component.absolute_path), components.len()); + moved_to.push(components.len()); + components.push(component); + } + } + } + for component in &mut components { + component.parent = component.parent.map(|parent| moved_to[parent]); + } + ImportPlan { + components, + warnings: plan.warnings, + } +} + +/// Windows treats paths case-insensitively and qrate stores them with `/`, so compare them that +/// way rather than as raw `Path`s. +fn comparable(path: &Path) -> String { + normalized_path(path).to_lowercase() +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use crate::duplicates::{DuplicatePolicy, ExistingComponent, Parent, Resolution, resolve}; + use crate::{EntryKind, ImportPlan, PlannedComponent, Warning}; + + /// The host's rule, reproduced here so the tests do not depend on `settings`. + fn file_keys(path: &str) -> Vec { + let lower = path.to_lowercase(); + let name = lower.rsplit('/').next().unwrap_or(&lower).to_string(); + let stem = name.rsplit_once('.').map(|(stem, _)| stem).unwrap_or(&name); + vec![name.clone(), stem.to_string()] + } + + fn component(path: &str, kind: EntryKind, parent: Option) -> PlannedComponent { + PlannedComponent { + title: path.rsplit('/').next().unwrap_or(path).into(), + absolute_path: PathBuf::from(path), + source_path: PathBuf::from(path), + kind, + parent, + level_key: "item".into(), + } + } + + fn plan(components: Vec) -> ImportPlan { + ImportPlan { + components, + warnings: Vec::new(), + } + } + + fn existing(key: u64, source: Option<&str>, keys: &[&str]) -> ExistingComponent { + ExistingComponent { + key, + absolute_source: source.map(PathBuf::from), + filename_keys: keys.iter().map(|key| (*key).to_string()).collect(), + } + } + + #[test] + fn the_same_stored_file_is_a_duplicate_under_every_policy() { + let plan = || plan(vec![component("C:/a/one.jpg", EntryKind::File, None)]); + let existing = [existing(7, Some("c:/A/one.jpg"), &[])]; + + for (policy, expected) in [ + (DuplicatePolicy::Skip, Resolution::Skip { existing: 7 }), + (DuplicatePolicy::Update, Resolution::Update { existing: 7 }), + (DuplicatePolicy::AddAsNew, Resolution::Create), + ] { + let resolved = resolve(plan(), &existing, file_keys, policy); + assert_eq!(resolved.resolutions, [expected], "{}", policy.key()); + } + } + + #[test] + fn one_filename_match_claims_the_row_and_several_stay_ambiguous() { + let resolved = resolve( + plan(vec![ + component("C:/drop/one.jpg", EntryKind::File, None), + component("C:/drop/two.jpg", EntryKind::File, None), + ]), + &[ + existing(1, None, &["one.jpg", "one"]), + existing(2, None, &["two.jpg", "two"]), + existing(3, None, &["two.jpg", "two"]), + ], + file_keys, + DuplicatePolicy::Skip, + ); + assert_eq!( + resolved.resolutions, + [ + Resolution::Skip { existing: 1 }, + Resolution::Ambiguous { + candidates: vec![2, 3] + } + ] + ); + assert_eq!(resolved.ambiguous(), 1); + } + + #[test] + fn a_folder_matches_only_its_own_path() { + let resolved = resolve( + plan(vec![component( + "C:/drop/Photographs", + EntryKind::Directory, + None, + )]), + &[existing(1, None, &["photographs"])], + file_keys, + DuplicatePolicy::Skip, + ); + assert_eq!(resolved.resolutions, [Resolution::Create]); + } + + #[test] + fn new_files_land_inside_the_folder_the_project_already_has() { + let resolved = resolve( + plan(vec![ + component("C:/drop/Series", EntryKind::Directory, None), + component("C:/drop/Series/old.jpg", EntryKind::File, Some(0)), + component("C:/drop/Series/new.jpg", EntryKind::File, Some(0)), + ]), + &[ + existing(10, Some("C:/drop/Series"), &[]), + existing(11, Some("C:/drop/Series/old.jpg"), &[]), + ], + file_keys, + DuplicatePolicy::Skip, + ); + assert_eq!( + resolved.resolutions, + [ + Resolution::Skip { existing: 10 }, + Resolution::Skip { existing: 11 }, + Resolution::Create, + ] + ); + // The genuinely new file belongs to the series already in the project, not to a second + // copy of it. + assert_eq!( + resolved.parents, + [None, Some(Parent::Existing(10)), Some(Parent::Existing(10))] + ); + assert_eq!((resolved.created(), resolved.duplicates()), (1, 2)); + } + + #[test] + fn a_path_reached_twice_in_one_import_collapses_without_asking() { + let mut duplicated = plan(vec![ + component("C:/drop/Series", EntryKind::Directory, None), + component("C:/drop/Series/one.jpg", EntryKind::File, Some(0)), + component("C:/drop/Series/one.jpg", EntryKind::File, Some(0)), + ]); + duplicated + .warnings + .push(Warning::UnreadableDirectory("C:/x".into())); + + let resolved = resolve(duplicated, &[], file_keys, DuplicatePolicy::Skip); + assert_eq!(resolved.plan.components.len(), 2); + assert_eq!(resolved.plan.warnings.len(), 1); + assert_eq!(resolved.created(), 2); + } +} diff --git a/crates/file-ingest/src/lib.rs b/crates/file-ingest/src/lib.rs new file mode 100644 index 00000000..844866dc --- /dev/null +++ b/crates/file-ingest/src/lib.rs @@ -0,0 +1,436 @@ +//! Filesystem discovery for project creation and live imports. +//! +//! This crate only inventories paths. It does not decide which qrate columns receive them or +//! mutate a project, so the wizard, table, and future folder watcher can share one traversal. + +pub mod duplicates; + +use std::fs; +use std::path::{Path, PathBuf}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum EntryKind { + File, + Directory, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Entry { + /// The discovered path joined below the supplied root. + pub path: PathBuf, + /// Path below the supplied root. Directory entries never include the root itself. + pub relative_path: PathBuf, + pub kind: EntryKind, + /// Index of the containing directory in [`Inventory::entries`]. `None` means the root. + pub parent: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Warning { + UnreadableDirectory(PathBuf), + DirectorySymlinkSkipped(PathBuf), +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Inventory { + pub entries: Vec, + pub warnings: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PlannedComponent { + pub title: String, + pub absolute_path: PathBuf, + pub source_path: PathBuf, + pub kind: EntryKind, + pub parent: Option, + pub level_key: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ImportPlan { + pub components: Vec, + pub warnings: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PlanOptions<'a> { + pub recursive: bool, + /// A dropped directory is itself an archival component. A folder merely selected as the + /// project's file root can opt out and import only its contents. + pub include_root: bool, + pub folder_level_key: &'a str, + pub file_level_key: &'a str, +} + +impl Default for PlanOptions<'_> { + fn default() -> Self { + Self { + recursive: true, + include_root: true, + folder_level_key: "series", + file_level_key: "item", + } + } +} + +impl Inventory { + pub fn files(&self) -> impl Iterator { + self.entries + .iter() + .filter(|entry| entry.kind == EntryKind::File) + } +} + +#[derive(Debug)] +pub enum Error { + NotDirectory, + Permission(std::io::Error), + Empty, +} + +/// Inventory `root` in normalized lexical order. +/// +/// A non-recursive scan returns files directly under the root. A recursive scan also returns +/// directory components and their descendants. Directory symlinks are not followed because an +/// import must terminate even when the filesystem contains a cycle. +pub fn scan(root: &Path, recursive: bool) -> Result { + if !root.is_dir() { + return Err(Error::NotDirectory); + } + fs::read_dir(root).map_err(Error::Permission)?; + + let mut inventory = Inventory::default(); + visit(root, root, None, recursive, &mut inventory); + if inventory.files().next().is_none() { + return Err(Error::Empty); + } + Ok(inventory) +} + +/// Produces a side-effect-free component tree suitable for an import preview. +pub fn plan(root: &Path, options: &PlanOptions<'_>) -> Result { + let inventory = scan(root, options.recursive)?; + let root_offset = usize::from(options.include_root); + let mut components = Vec::with_capacity(inventory.entries.len() + root_offset); + if options.include_root { + components.push(PlannedComponent { + title: display_name(root), + absolute_path: root.to_path_buf(), + source_path: PathBuf::new(), + kind: EntryKind::Directory, + parent: None, + level_key: options.folder_level_key.to_string(), + }); + } + components.extend(inventory.entries.iter().map(|entry| { + PlannedComponent { + title: display_name(&entry.path), + absolute_path: entry.path.clone(), + source_path: entry.relative_path.clone(), + kind: entry.kind, + parent: entry + .parent + .map(|parent| parent + root_offset) + .or(options.include_root.then_some(0)), + level_key: match entry.kind { + EntryKind::Directory => options.folder_level_key, + EntryKind::File => options.file_level_key, + } + .to_string(), + } + })); + Ok(ImportPlan { + components, + warnings: inventory.warnings, + }) +} + +/// Plans one or more dropped files and directories as independent root components. +pub fn plan_paths(paths: &[PathBuf], options: &PlanOptions<'_>) -> Result { + if paths.is_empty() { + return Err(Error::Empty); + } + let mut combined = ImportPlan { + components: Vec::new(), + warnings: Vec::new(), + }; + for path in paths { + if path.is_dir() { + let mut next = plan(path, options)?; + let offset = combined.components.len(); + for component in &mut next.components { + component.parent = component.parent.map(|parent| parent + offset); + } + combined.components.extend(next.components); + combined.warnings.extend(next.warnings); + } else if path.is_file() { + combined.components.push(PlannedComponent { + title: display_name(path), + absolute_path: path.clone(), + source_path: path.clone(), + kind: EntryKind::File, + parent: None, + level_key: options.file_level_key.to_string(), + }); + } else { + return Err(Error::NotDirectory); + } + } + if combined.components.is_empty() { + return Err(Error::Empty); + } + Ok(combined) +} + +fn display_name(path: &Path) -> String { + path.file_name() + .map(|name| name.to_string_lossy().into_owned()) + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| normalized_path(path)) +} + +fn visit( + root: &Path, + directory: &Path, + parent: Option, + recursive: bool, + inventory: &mut Inventory, +) { + let entries = match fs::read_dir(directory) { + Ok(entries) => entries, + Err(_) => { + inventory + .warnings + .push(Warning::UnreadableDirectory(directory.to_path_buf())); + return; + } + }; + let mut paths: Vec<_> = entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .collect(); + paths.sort_by(|left, right| { + let (left, right) = (relative_string(root, left), relative_string(root, right)); + left.to_lowercase() + .cmp(&right.to_lowercase()) + .then_with(|| left.cmp(&right)) + }); + + for path in paths { + let Some(name) = path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + if is_ignored(name) { + continue; + } + + let Ok(file_type) = fs::symlink_metadata(&path).map(|meta| meta.file_type()) else { + continue; + }; + if file_type.is_symlink() && path.is_dir() { + inventory + .warnings + .push(Warning::DirectorySymlinkSkipped(path)); + continue; + } + if path.is_dir() { + if recursive { + let index = inventory.entries.len(); + inventory.entries.push(Entry { + relative_path: path.strip_prefix(root).unwrap_or(&path).to_path_buf(), + path: path.clone(), + kind: EntryKind::Directory, + parent, + }); + visit(root, &path, Some(index), true, inventory); + } + } else if path.is_file() { + inventory.entries.push(Entry { + relative_path: path.strip_prefix(root).unwrap_or(&path).to_path_buf(), + path, + kind: EntryKind::File, + parent, + }); + } + } +} + +fn relative_string(root: &Path, path: &Path) -> String { + path.strip_prefix(root) + .unwrap_or(path) + .to_string_lossy() + .replace('\\', "/") +} + +pub fn normalized_path(path: &Path) -> String { + path.to_string_lossy().replace('\\', "/") +} + +pub fn is_ignored(name: &str) -> bool { + name.starts_with("._") || name.eq_ignore_ascii_case(".ds_store") +} + +#[cfg(test)] +mod tests { + use std::fs; + + use crate::{EntryKind, Error, PlanOptions, normalized_path, plan, plan_paths, scan}; + + #[test] + fn recursive_inventory_preserves_directories_parents_and_duplicate_names() { + let root = tempfile::tempdir().unwrap(); + fs::create_dir(root.path().join("a")).unwrap(); + fs::create_dir(root.path().join("b")).unwrap(); + fs::write(root.path().join("a").join("same.jpg"), "a").unwrap(); + fs::write(root.path().join("b").join("same.jpg"), "b").unwrap(); + + let inventory = scan(root.path(), true).unwrap(); + let paths: Vec<_> = inventory + .entries + .iter() + .map(|entry| { + ( + normalized_path(&entry.relative_path), + entry.kind, + entry.parent, + ) + }) + .collect(); + assert_eq!( + paths, + vec![ + ("a".into(), EntryKind::Directory, None), + ("a/same.jpg".into(), EntryKind::File, Some(0)), + ("b".into(), EntryKind::Directory, None), + ("b/same.jpg".into(), EntryKind::File, Some(2)), + ] + ); + } + + #[test] + fn non_recursive_inventory_returns_only_root_files() { + let root = tempfile::tempdir().unwrap(); + fs::create_dir(root.path().join("nested")).unwrap(); + fs::write(root.path().join("root.jpg"), "root").unwrap(); + fs::write(root.path().join("nested").join("inside.jpg"), "inside").unwrap(); + + let inventory = scan(root.path(), false).unwrap(); + let paths: Vec<_> = inventory + .files() + .map(|entry| normalized_path(&entry.relative_path)) + .collect(); + assert_eq!(paths, ["root.jpg"]); + } + + #[test] + fn os_cruft_does_not_make_a_folder_nonempty() { + let root = tempfile::tempdir().unwrap(); + fs::write(root.path().join(".DS_Store"), "x").unwrap(); + fs::write(root.path().join("._photo.jpg"), "x").unwrap(); + assert!(matches!(scan(root.path(), true), Err(Error::Empty))); + } + + #[test] + fn ordering_is_case_insensitive_and_stable() { + let root = tempfile::tempdir().unwrap(); + fs::write(root.path().join("z.jpg"), "z").unwrap(); + fs::write(root.path().join("A.jpg"), "a").unwrap(); + let paths: Vec<_> = scan(root.path(), true) + .unwrap() + .files() + .map(|entry| normalized_path(&entry.relative_path)) + .collect(); + assert_eq!(paths, ["A.jpg", "z.jpg"]); + } + + #[test] + fn import_plan_includes_a_dropped_root_and_maps_levels() { + let root = tempfile::tempdir().unwrap(); + let collection = root.path().join("Photographs"); + fs::create_dir_all(collection.join("Events")).unwrap(); + fs::write(collection.join("Events").join("001.jpg"), "photo").unwrap(); + + let plan = plan( + &collection, + &PlanOptions { + folder_level_key: "series", + file_level_key: "item", + ..Default::default() + }, + ) + .unwrap(); + let components: Vec<_> = plan + .components + .iter() + .map(|component| { + ( + component.title.as_str(), + component.kind, + component.parent, + component.level_key.as_str(), + normalized_path(&component.source_path), + ) + }) + .collect(); + assert_eq!( + components, + [ + ( + "Photographs", + EntryKind::Directory, + None, + "series", + "".into() + ), + ( + "Events", + EntryKind::Directory, + Some(0), + "series", + "Events".into() + ), + ( + "001.jpg", + EntryKind::File, + Some(1), + "item", + "Events/001.jpg".into(), + ), + ] + ); + } + + #[test] + fn selected_file_root_can_exclude_itself() { + let root = tempfile::tempdir().unwrap(); + fs::write(root.path().join("one.jpg"), "photo").unwrap(); + let plan = plan( + root.path(), + &PlanOptions { + include_root: false, + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(plan.components.len(), 1); + assert_eq!(plan.components[0].parent, None); + assert_eq!(plan.components[0].title, "one.jpg"); + } + + #[test] + fn several_dropped_roots_keep_independent_parent_indexes() { + let root = tempfile::tempdir().unwrap(); + let folder = root.path().join("folder"); + fs::create_dir(&folder).unwrap(); + fs::write(folder.join("inside.jpg"), "inside").unwrap(); + let loose = root.path().join("loose.jpg"); + fs::write(&loose, "loose").unwrap(); + + let plan = plan_paths(&[folder, loose], &PlanOptions::default()).unwrap(); + assert_eq!(plan.components.len(), 3); + assert_eq!(plan.components[0].parent, None); + assert_eq!(plan.components[1].parent, Some(0)); + assert_eq!(plan.components[2].parent, None); + } +} diff --git a/crates/preview/src/lib.rs b/crates/preview/src/lib.rs index cb2678e8..4f3b7c32 100644 --- a/crates/preview/src/lib.rs +++ b/crates/preview/src/lib.rs @@ -677,7 +677,13 @@ pub fn thumb(path: Option<&Path>, max_edge: u32, cx: &App) -> AnyElement { /// ever shows up in a memory profile. pub fn source(path: &Path, max_edge: u32, page: usize) -> ImageSource { let extension = extension(path).unwrap_or_default(); - if extension == "svg" || (max_edge == FULL && page == 0 && is_raster(&extension)) { + // ponytail: a GIF goes to gpui whole at every size, because only gpui's own decode keeps the + // frames that animate it. It skips the thumbnail cache and the memory budget, so a gallery of + // large GIFs holds them all; downscale every frame here if that shows up. + if extension == "svg" + || extension == "gif" + || (max_edge == FULL && page == 0 && is_raster(&extension)) + { return ImageSource::Resource(path.to_path_buf().into()); } let key = (path.to_path_buf(), max_edge, page); @@ -767,6 +773,10 @@ mod tests { assert!(native("/f/logo.svg", CARD, 0)); assert!(native("/f/logo.svg", FULL, 0)); + assert!( + native("/f/anim.gif", CARD, 0), + "the Details pane animates too" + ); // Capped sizes stay on the ladder — a card wants the shrunk, disk-cached copy. assert!(!native("/f/scan.jpg", CARD, 0)); // Formats gpui cannot read at all, at any size. diff --git a/crates/project-wizard/Cargo.toml b/crates/project-wizard/Cargo.toml index dd519278..6ede5a0a 100644 --- a/crates/project-wizard/Cargo.toml +++ b/crates/project-wizard/Cargo.toml @@ -14,8 +14,12 @@ gpui-component.workspace = true serde.workspace = true serde_json.workspace = true data-exchange = { path = "../data-exchange" } +file-ingest = { path = "../file-ingest" } diagnostics = { path = "../diagnostics" } # Column configs can carry a plugin's mapping, and this is where its shape lives. plugin-api = { path = "../plugin-api" } settings = { path = "../settings" } window-wrapper = { path = "../window-wrapper" } + +[dev-dependencies] +tempfile.workspace = true diff --git a/crates/project-wizard/src/column_config.rs b/crates/project-wizard/src/column_config.rs new file mode 100644 index 00000000..568df100 --- /dev/null +++ b/crates/project-wizard/src/column_config.rs @@ -0,0 +1,401 @@ +//! Loading a column config — a file or a Google Sheet listing each column's type, description and +//! checks. One view for both places it happens: the wizard's Columns step, before a project exists, +//! and an open project, where it re-types the columns already there. + +use gpui::{prelude::FluentBuilder, *}; +use gpui_component::button::Button; +use gpui_component::dialog::DialogButtonProps; +use gpui_component::input::{Input, InputState}; +use gpui_component::label::Label; +use gpui_component::tab::{Tab, TabBar}; +use gpui_component::{ActiveTheme, WindowExt, h_flex, v_flex}; + +use plugin_api::{ColumnMapContributions, ColumnMapSpec}; +use settings::columns::ColumnSettings; +use settings::project::CurrentProject; + +use crate::data::{self, ColumnConfigEntry, ColumnConfigPreview}; +use crate::steps::files::{MsgKind, inline_message}; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum LoadConfigTab { + File, + Sheet, +} + +/// The source picker, what it loaded, and how that lines up with `headers`. It renders itself, so +/// a dialog can hold it as a child and never needs rebuilding when a click changes what it shows. +pub struct ColumnConfigLoader { + pub(crate) headers: Vec, + tab: LoadConfigTab, + file_path: String, + sheet_link: Entity, + pub(crate) preview: Option, + error: Option, +} + +impl ColumnConfigLoader { + pub fn new(headers: Vec, window: &mut Window, cx: &mut Context) -> Self { + Self { + headers, + tab: LoadConfigTab::File, + file_path: String::new(), + sheet_link: cx.new(|cx| { + InputState::new(window, cx).placeholder("docs.google.com/spreadsheets/d/…") + }), + preview: None, + error: None, + } + } + + fn loaded(&mut self, result: Result) { + (self.preview, self.error) = match result { + Ok(preview) => (Some(preview), None), + Err(message) => (None, Some(message.into())), + }; + } + + fn load_path(&mut self, path: &std::path::Path, cx: &mut Context) { + let path = path.to_string_lossy().to_string(); + let result = data::load_column_config(&path, &self.headers); + self.file_path = path; + self.loaded(result.map_err(|error| error.message())); + cx.notify(); + } + + fn browse(&mut self, window: &mut Window, cx: &mut Context) { + let receiver = cx.prompt_for_paths(PathPromptOptions { + files: true, + directories: false, + multiple: false, + prompt: Some("Choose a column config file".into()), + }); + cx.spawn_in(window, async move |this, cx| { + if let Ok(Ok(Some(paths))) = receiver.await + && let Some(path) = paths.first() + { + let path = path.to_string_lossy().to_string(); + this.update(cx, |this, cx| { + this.load_path(std::path::Path::new(&path), cx); + }) + .ok(); + } + }) + .detach(); + } + + fn check_sheet(&mut self, cx: &mut Context) { + let link = self.sheet_link.read(cx).value().to_string(); + let headers = self.headers.clone(); + let fetch = cx + .background_executor() + .spawn(async move { data_exchange::fetch_sheet(&link) }); + cx.spawn(async move |this, cx| { + let result = fetch + .await + .map_err(|error| error.to_string()) + .and_then(|sheet| { + data::parse_column_config(sheet.headers, sheet.rows, &headers) + .map_err(|error| error.message()) + }); + this.update(cx, |this, cx| { + this.loaded(result); + cx.notify(); + }) + .ok(); + }) + .detach(); + } +} + +impl Render for ColumnConfigLoader { + fn render(&mut self, _window: &mut Window, cx: &mut Context) -> impl IntoElement { + let tabs = TabBar::new("load-config-tabs") + .segmented() + .selected_index(match self.tab { + LoadConfigTab::File => 0, + LoadConfigTab::Sheet => 1, + }) + .on_click(cx.listener(|this, ix: &usize, _, cx| { + this.tab = match ix { + 0 => LoadConfigTab::File, + _ => LoadConfigTab::Sheet, + }; + cx.notify(); + })) + .child(Tab::new().label("From file")) + .child(Tab::new().label("From Google Sheet")); + + let body = match self.tab { + LoadConfigTab::File => v_flex() + .gap_1() + .child(Label::new("Config file").text_sm()) + .child( + h_flex() + .gap_2() + .child( + div() + .flex_1() + .min_w(px(0.)) + .px_2() + .py_1p5() + .rounded_md() + .border_1() + .border_color(cx.theme().border) + .text_sm() + .text_color(cx.theme().muted_foreground) + .child(match self.file_path.is_empty() { + true => "Choose a file…".to_string(), + false => self.file_path.clone(), + }), + ) + .child( + Button::new("browse-config") + .label("Browse…") + .outline() + .on_click( + cx.listener(|this, _, window, cx| this.browse(window, cx)), + ), + ), + ), + LoadConfigTab::Sheet => v_flex() + .gap_1() + .child(Label::new("Sheet link").text_sm()) + .child( + h_flex() + .gap_2() + .child(Input::new(&self.sheet_link).flex_1()) + .child( + Button::new("check-config-sheet") + .label("Check") + .outline() + .on_click(cx.listener(|this, _, _, cx| this.check_sheet(cx))), + ), + ), + }; + + v_flex() + .drag_over::(|style, _, _, cx| style.bg(cx.theme().secondary_hover)) + .on_drop( + cx.listener(|this, paths: &ExternalPaths, _, cx| match paths.paths() { + [path] if path.is_file() => this.load_path(path, cx), + _ => { + this.preview = None; + this.error = Some("Drop one supported column config file.".into()); + cx.notify(); + } + }), + ) + .gap_3() + .child(tabs) + .child(body) + .map(|el| match (&self.preview, &self.error) { + (Some(preview), _) => el.child(inline_message( + "config-status", + format!("Loaded {} columns", preview.entries.len()), + MsgKind::Success, + )), + (None, Some(error)) => el.child(inline_message( + "config-status", + error.clone(), + MsgKind::Error, + )), + (None, None) => el, + }) + .when(self.preview.is_some(), |el| { + el.child(mapping(&self.headers, self.preview.as_ref(), cx)) + }) + } +} + +/// Each column beside the config entry it takes its type from, with the ones the config says +/// nothing about marked. +pub(crate) fn mapping( + headers: &[String], + config: Option<&ColumnConfigPreview>, + cx: &App, +) -> AnyElement { + let headers = if headers.is_empty() { + config + .map(|config| { + config + .entries + .iter() + .map(|entry| entry.name.clone()) + .collect::>() + }) + .unwrap_or_default() + } else { + headers.to_vec() + }; + v_flex() + .gap_1() + .child( + Label::new("Column → qrate field") + .text_sm() + .text_color(cx.theme().muted_foreground), + ) + .when(headers.is_empty(), |table| { + table.child( + Label::new("No columns yet.") + .text_sm() + .text_color(cx.theme().muted_foreground), + ) + }) + .children(headers.iter().map(|header| { + let target = + config + .and_then(|config| entry_for(config, header)) + .map(|entry| match entry.data_type.is_empty() { + true => entry.name.clone(), + false => format!("{} ({})", entry.name, entry.data_type), + }); + h_flex() + .justify_between() + .text_sm() + .py_1() + .border_b_1() + .border_color(cx.theme().border) + // The warning colour already says "unmapped" — a trailing glyph saying it + // again depends on font coverage for nothing. + .when(target.is_none(), |el| el.text_color(cx.theme().warning)) + .child(header.clone()) + .child(format!("→ {}", target.as_deref().unwrap_or("Unmapped"))) + })) + .into_any_element() +} + +pub(crate) fn entry_for<'a>( + config: &'a ColumnConfigPreview, + header: &str, +) -> Option<&'a ColumnConfigEntry> { + config + .entries + .iter() + .find(|entry| entry.name.eq_ignore_ascii_case(header)) +} + +/// Lay what `entry` says over `settings`. Only what the file states is written, so a config with no +/// Spellcheck column leaves each column's spell-checking where the archivist put it. +pub(crate) fn apply_entry( + entry: &ColumnConfigEntry, + maps: &[(SharedString, ColumnMapSpec)], + settings: &mut ColumnSettings, +) { + if entry.authority.is_some() { + settings.authority = entry.authority.clone(); + } + if let Some(on) = entry.spellcheck { + settings.spellcheck = on; + } + if let Some(on) = entry.variant_review { + settings.variant_review = on; + } + // Severity is per producer: the authority names its own, and each plugin's rides in the + // `::Severity` column beside its mapping. + if let (Some(authority), Some(severity)) = + (entry.authority.clone(), entry.authority_severity.clone()) + { + settings.severity.insert(authority, severity); + } + for (plugin, spec) in maps { + if let Some(cell) = entry.extra.get(&format!("{plugin}::{}", spec.label)) { + let chosen: Vec = cell + .split(';') + .map(str::trim) + .filter(|v| !v.is_empty()) + .map(|v| v.to_string().into()) + .collect(); + ColumnMapContributions::put(plugin, spec, settings, &chosen); + } + if let Some(severity) = entry.extra.get(&format!("{plugin}::Severity")) { + settings + .severity + .insert(plugin.to_string(), severity.to_ascii_lowercase()); + } + } +} + +/// Load a column config onto the open project: every column the config names takes its type, +/// description and checks. Columns it doesn't name are left exactly as they are. +pub fn open_column_config_dialog(window: &mut Window, cx: &mut App) { + let Some(headers) = cx + .try_global::() + .map(|project| project.data.headers.clone()) + else { + return; + }; + let loader = cx.new(|cx| ColumnConfigLoader::new(headers, window, cx)); + window.open_dialog(cx, move |dialog, _, _| { + let apply_from = loader.clone(); + dialog + .title("Load column config") + .child(loader.clone()) + .button_props( + DialogButtonProps::default() + .ok_text("Apply") + .show_cancel(true), + ) + .on_ok(move |_, _, cx| { + let Some(config) = apply_from.read(cx).preview.clone() else { + return false; + }; + apply(&config, cx); + true + }) + }); +} + +fn apply(config: &ColumnConfigPreview, cx: &mut App) { + let headers = cx.global::().data.headers.clone(); + let maps = ColumnMapContributions::all(cx); + let mut applied = 0; + for header in &headers { + let Some(entry) = entry_for(config, header) else { + continue; + }; + CurrentProject::set_column_type(header, &entry.data_type, cx); + if !entry.description.is_empty() { + CurrentProject::set_column_notes(header, &entry.description, cx); + } + settings::columns::update(header, |settings| apply_entry(entry, &maps, settings), cx); + applied += 1; + } + log::info!( + "applied a column config to {applied} of {} columns", + headers.len() + ); +} + +#[cfg(test)] +mod tests { + use settings::columns::ColumnSettings; + + use crate::column_config::apply_entry; + use crate::data::ColumnConfigEntry; + + /// Re-applying a config to an open project must not undo choices the file is silent on. + #[test] + fn only_what_the_config_states_is_overwritten() { + let mut settings = ColumnSettings { + spellcheck: false, + authority: Some("LCSH".into()), + ..Default::default() + }; + apply_entry( + &ColumnConfigEntry { + name: "Subject".into(), + variant_review: Some(true), + ..Default::default() + }, + &[], + &mut settings, + ); + assert!( + !settings.spellcheck, + "no Spellcheck column, so it stays off" + ); + assert_eq!(settings.authority.as_deref(), Some("LCSH")); + assert!(settings.variant_review); + } +} diff --git a/crates/project-wizard/src/data.rs b/crates/project-wizard/src/data.rs index 8d8d6d6a..d9daacb8 100644 --- a/crates/project-wizard/src/data.rs +++ b/crates/project-wizard/src/data.rs @@ -4,7 +4,6 @@ //! [`match_folder`] path from here on. use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fs; use std::path::Path; use calamine::{Data, Reader, open_workbook_auto}; @@ -112,6 +111,8 @@ pub struct FolderMatch { /// turns each one into an otherwise-empty row so the file is in the collection instead of /// silently left behind. pub extra_files: Vec, + /// Files that several rows claim. They are never linked automatically. + pub ambiguous_files: usize, } /// Why a files folder was turned down. Each variant carries what the archivist needs to act on @@ -184,44 +185,29 @@ impl FolderError { /// wizard's check has to be able to look as deep as the app will. pub fn list_files(folder: &str, recursive: bool) -> Result, FolderError> { let dir = Path::new(folder); - if !dir.is_dir() { - log::warn!("Files folder rejected: {folder} is not a folder qrate can see"); - return Err(FolderError::NotFound); - } - // Reading the root up front keeps an unreadable folder an error; an unreadable subfolder - // below is only skipped. - fs::read_dir(dir).map_err(|e| { - log::warn!("Files folder rejected: cannot read {folder} — {e}"); - FolderError::Permission - })?; - - let mut files = Vec::new(); - let mut stack = vec![dir.to_path_buf()]; - while let Some(dir) = stack.pop() { - let Ok(entries) = fs::read_dir(&dir) else { - continue; - }; - for entry in entries.filter_map(|e| e.ok()) { - let path = entry.path(); - let Some(name) = path.file_name().and_then(|n| n.to_str()) else { - continue; - }; - if name.starts_with("._") || name.eq_ignore_ascii_case(".ds_store") { - continue; - } - if path.is_dir() { - if recursive { - stack.push(path); - } - } else { - files.push(name.to_string()); - } + let inventory = file_ingest::scan(dir, recursive).map_err(|error| match error { + file_ingest::Error::NotDirectory => { + log::warn!("Files folder rejected: {folder} is not a folder qrate can see"); + FolderError::NotFound } - } - if files.is_empty() { - log::warn!("Files folder rejected: {folder} holds no files (recursive: {recursive})"); - return Err(FolderError::Empty { recursive }); - } + file_ingest::Error::Permission(error) => { + log::warn!("Files folder rejected: cannot read {folder} — {error}"); + FolderError::Permission + } + file_ingest::Error::Empty => { + log::warn!("Files folder rejected: {folder} holds no files (recursive: {recursive})"); + FolderError::Empty { recursive } + } + })?; + for warning in &inventory.warnings { + log::warn!("Files folder scan warning for {folder}: {warning:?}"); + } + // Relative paths keep same-named files in separate subfolders distinct. Filename matching + // still accepts them because `settings::filenames` reads the path tail as one of its keys. + let files: Vec<_> = inventory + .files() + .map(|entry| file_ingest::normalized_path(&entry.relative_path)) + .collect(); log::debug!( "Files folder {folder}: {} file(s) found (recursive: {recursive})", files.len() @@ -236,6 +222,7 @@ pub fn inventory_folder(folder: &str, recursive: bool) -> Result = Vec::new(); let mut used_files: HashSet = HashSet::new(); + let mut claims: HashMap = HashMap::new(); for row in &preview.rows { let hit = row.iter().find_map(|cell| { settings::filenames::lookup_keys(cell) @@ -270,6 +258,9 @@ pub fn match_folder( Some(hit) => { matched_rows += 1; used_files.extend(hit); + for file in hit { + *claims.entry(*file).or_default() += 1; + } } // The row's first non-empty cell is what a person recognizes the row by, and the // value most likely meant to name a file. @@ -312,6 +303,7 @@ pub fn match_folder( Ok(FolderMatch { matched_rows, + ambiguous_files: claims.values().filter(|rows| **rows > 1).count(), total_rows: preview.rows.len(), extra_files: files .into_iter() @@ -421,6 +413,14 @@ pub fn load_column_config( _ => ColumnConfigError::Io("it isn't a CSV, TSV, Excel or ODS file we can read".into()), })?; + parse_column_config(headers, records, against_headers) +} + +pub(crate) fn parse_column_config( + headers: Vec, + records: Vec>, + against_headers: &[String], +) -> Result { let at = |wanted: &str| headers.iter().position(|h| h.eq_ignore_ascii_case(wanted)); let (name_ix, type_ix) = (at("Column Name"), at("Data Type")); let (desc_ix, authority_ix) = (at("Description"), at("Authority")); @@ -486,6 +486,7 @@ pub fn load_column_config( #[cfg(test)] mod tests { use super::*; + use std::fs; use std::io::Write; fn sample_dir() -> std::path::PathBuf { @@ -548,7 +549,7 @@ mod tests { ); let mut deep = list_files(dir.to_str().unwrap(), true).unwrap(); deep.sort(); - assert_eq!(deep, ["nested.jpg", "top.jpg"]); + assert_eq!(deep, ["batch/nested.jpg", "top.jpg"]); } #[test] @@ -562,7 +563,7 @@ mod tests { inventory.extra_files.sort(); assert_eq!(inventory.matched_rows, 0); assert_eq!(inventory.total_rows, 0); - assert_eq!(inventory.extra_files, ["one.jpg", "two.png"]); + assert_eq!(inventory.extra_files, ["batch/two.png", "one.jpg"]); } #[test] @@ -663,6 +664,30 @@ mod tests { load_column_config(unrelated.to_str().unwrap(), &headers), Err(ColumnConfigError::NoMatch) )); + let blank_preview = load_column_config(unrelated.to_str().unwrap(), &[]).unwrap(); + assert_eq!(blank_preview.entries[0].name, "Something Else"); + } + + #[test] + fn fetched_sheet_rows_use_the_same_column_config_shape_as_a_file() { + let preview = parse_column_config( + vec![ + "Column Name".into(), + "Data Type".into(), + "Description".into(), + ], + vec![vec![ + "Object Name".into(), + "Title".into(), + "Shown in the table".into(), + ]], + &[], + ) + .unwrap(); + + assert_eq!(preview.entries[0].name, "Object Name"); + assert_eq!(preview.entries[0].data_type, "Title"); + assert_eq!(preview.entries[0].description, "Shown in the table"); } /// A synonym is stored canonically so validators match one spelling; a type this build does diff --git a/crates/project-wizard/src/launcher.rs b/crates/project-wizard/src/launcher.rs index 4e27c780..e39b23c7 100644 --- a/crates/project-wizard/src/launcher.rs +++ b/crates/project-wizard/src/launcher.rs @@ -122,6 +122,105 @@ impl Launcher { self.recents = recent::list(cx); cx.notify(); } + + fn accept_drop( + &mut self, + paths: &[std::path::PathBuf], + window: &mut Window, + cx: &mut Context, + ) { + if paths.is_empty() { + return; + } + let is_project = |path: &std::path::Path| { + path.extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("qrate")) + }; + if paths.len() > 1 && paths.iter().any(|path| is_project(path)) { + let project = paths.iter().find(|path| is_project(path)).cloned(); + let material: Vec<_> = paths + .iter() + .filter(|path| !is_project(path)) + .cloned() + .collect(); + let answer = window.prompt( + PromptLevel::Info, + "Open a project or import material?", + Some("A project file and import material were dropped together."), + &["Open project", "Start project from material", "Cancel"], + cx, + ); + cx.spawn_in(window, async move |this, cx| { + match answer.await.unwrap_or(2) { + 0 => { + if let Some(project) = project { + this.update_in(cx, |this, window, cx| { + this.open_project_file( + project.to_string_lossy().into_owned(), + window, + cx, + ) + }) + .ok(); + } + } + 1 if !material.is_empty() => { + this.update_in(cx, |_, window, cx| { + wizard::open_project_wizard_seeded_paths( + EntryKind::Blank, + None, + material, + cx, + ); + window.remove_window(); + }) + .ok(); + } + _ => {} + } + }) + .detach(); + return; + } + let path = &paths[0]; + let extension = path + .extension() + .and_then(|extension| extension.to_str()) + .unwrap_or_default() + .to_ascii_lowercase(); + if paths.len() == 1 && extension == "qrate" { + self.open_project_file(path.to_string_lossy().into_owned(), window, cx); + return; + } + if paths.len() == 1 && path.is_dir() { + wizard::open_project_wizard_seeded_paths(EntryKind::Blank, None, paths.to_vec(), cx); + window.remove_window(); + return; + } + if paths.len() == 1 + && matches!( + extension.as_str(), + "csv" | "tsv" | "xlsx" | "xlsm" | "xlsb" | "xls" | "ods" + ) + { + wizard::open_project_wizard_seeded( + EntryKind::LocalFile, + Some(path.to_string_lossy().into_owned()), + None, + cx, + ); + window.remove_window(); + return; + } + if paths.iter().all(|path| path.is_file() || path.is_dir()) { + wizard::open_project_wizard_seeded_paths(EntryKind::Blank, None, paths.to_vec(), cx); + window.remove_window(); + } else { + self.error = Some("One or more dropped paths no longer exist.".into()); + cx.notify(); + } + } } impl Render for Launcher { @@ -191,6 +290,10 @@ impl Render for Launcher { v_flex() .size_full() .bg(cx.theme().background) + .drag_over::(|style, _, _, cx| style.bg(cx.theme().secondary_hover)) + .on_drop(cx.listener(|this, paths: &ExternalPaths, window, cx| { + this.accept_drop(paths.paths(), window, cx) + })) .child( TitleBar::new() .text_xs() diff --git a/crates/project-wizard/src/lib.rs b/crates/project-wizard/src/lib.rs index e54acd84..09f4b96d 100644 --- a/crates/project-wizard/src/lib.rs +++ b/crates/project-wizard/src/lib.rs @@ -3,6 +3,7 @@ //! Google Sheet entry paths. See `crates/project-wizard/../../chats/chat1.md` //! in the design handoff bundle for the UX rationale behind each step. +mod column_config; mod data; pub mod launcher; mod project; @@ -10,5 +11,6 @@ mod recent; mod steps; pub mod wizard; +pub use column_config::open_column_config_dialog; pub use launcher::{LauncherHooks, open_launcher_window}; pub use wizard::{EntryKind, open_project_wizard}; diff --git a/crates/project-wizard/src/project.rs b/crates/project-wizard/src/project.rs index 34b89881..2ff0cad0 100644 --- a/crates/project-wizard/src/project.rs +++ b/crates/project-wizard/src/project.rs @@ -5,7 +5,10 @@ use std::path::Path; -pub use settings::project::{ProjectColumn, ProjectSpec, RowId, StoredNote, write_notes}; +pub use settings::project::{ + ProjectColumn, ProjectSpec, RowId, RowStructure, SourceKind, StoredNote, write_notes, + write_row_structure, +}; pub fn sanitize_file_stem(name: &str) -> String { let cleaned: String = name diff --git a/crates/project-wizard/src/steps/columns.rs b/crates/project-wizard/src/steps/columns.rs index e6fe1344..92b60fb6 100644 --- a/crates/project-wizard/src/steps/columns.rs +++ b/crates/project-wizard/src/steps/columns.rs @@ -2,21 +2,17 @@ //! from the "load from file/Sheet" card. use gpui::{prelude::FluentBuilder, *}; -use gpui_component::button::Button; use gpui_component::collapsible::Collapsible; use gpui_component::combobox::{Combobox, ComboboxState}; use gpui_component::dialog::DialogButtonProps; -use gpui_component::input::Input; use gpui_component::label::Label; use gpui_component::searchable_list::SearchableVec; -use gpui_component::tab::{Tab, TabBar}; -use gpui_component::{ActiveTheme, IndexPath, Sizable, StyledExt, WindowExt, h_flex, v_flex}; +use gpui_component::{ActiveTheme, IndexPath, Sizable, StyledExt, WindowExt, v_flex}; +use crate::column_config::{ColumnConfigLoader, mapping}; use crate::data; use crate::steps::files::{MsgKind, inline_message}; -use crate::wizard::{ - ColumnChoice, ColumnSource, EntryKind, LoadConfigTab, ProjectWizard, option_card, -}; +use crate::wizard::{ColumnChoice, ColumnSource, EntryKind, ProjectWizard, option_card}; fn sync_required_picker( state: &Entity>>, @@ -66,11 +62,7 @@ fn inferred_required_column( } impl ProjectWizard { fn inferred_required_column(&self, kind: settings::columns::ColumnType) -> Option { - inferred_required_column( - &self.spreadsheet_headers(), - self.config_preview.as_ref(), - kind, - ) + inferred_required_column(&self.effective_headers(), self.selected_config(), kind) } fn prefill_required_columns(&mut self) { @@ -99,7 +91,7 @@ impl ProjectWizard { value: "".into(), label: "Choose a column…".into(), }) - .chain(self.spreadsheet_headers().into_iter().map(|header| { + .chain(self.effective_headers().into_iter().map(|header| { let label: SharedString = header.clone().into(); ColumnChoice { value: header.into(), @@ -179,209 +171,29 @@ impl ProjectWizard { )) }) } - fn browse_config_file(&mut self, window: &mut Window, cx: &mut Context) { - let receiver = cx.prompt_for_paths(PathPromptOptions { - files: true, - directories: false, - multiple: false, - prompt: Some("Choose a column config file".into()), - }); - cx.spawn_in(window, async move |this, cx| { - if let Ok(Ok(Some(paths))) = receiver.await - && let Some(path) = paths.first() - { - let s = path.to_string_lossy().to_string(); - this.update_in(cx, |this, window, cx| { - this.load_config_from_file(s, cx); - let entity = cx.entity(); - this.refresh_load_config_dialog(entity, window, cx); - }) - .ok(); - } - }) - .detach(); - } - - fn load_config_from_file(&mut self, path: String, _cx: &mut Context) { - self.config_file_path = path; - let headers = self.spreadsheet_headers(); - match data::load_column_config(&self.config_file_path, &headers) { - Ok(preview) => { - self.config_preview = Some(preview); - self.config_error = None; - self.reset_required_column_defaults(); - } - Err(e) => { - self.config_error = Some(e.message().into()); - self.config_preview = None; - } - } - } - - /// Fetches the given public Sheet as `.xlsx` and turns its headers into a - /// column config (one Text column each). - // move to the background executor if it ever drags. - fn load_config_from_sheet(&mut self, cx: &mut Context) { - let link = self.sheet_link_input.read(cx).value().to_string(); - let fetched = data_exchange::fetch_sheet(&link) - .map_err(|e| e.to_string()) - .map(data_exchange::SpreadsheetPreview::from); - match fetched { - Ok(preview) => { - self.config_preview = Some(data::ColumnConfigPreview { - entries: preview - .headers - .into_iter() - .map(|name| data::ColumnConfigEntry { - name, - data_type: "Text".into(), - ..Default::default() - }) - .collect(), - }); - self.config_error = None; - self.reset_required_column_defaults(); + fn open_load_config_dialog(&mut self, window: &mut Window, cx: &mut Context) { + let headers = if self.entry_kind == EntryKind::Blank { + Vec::new() + } else { + self.spreadsheet_headers() + }; + let loader = match &self.config_loader { + // The spreadsheet may have changed since, and a config is checked against its headers. + Some(loader) => { + loader.update(cx, |loader, _| loader.headers = headers); + loader.clone() } - Err(msg) => { - self.config_error = Some(msg.into()); - self.config_preview = None; + None => { + let loader = cx.new(|cx| ColumnConfigLoader::new(headers, window, cx)); + self.config_loader.insert(loader).clone() } - } - } - - /// Registers the dialog with a builder that closes over a *snapshot* of - /// the fields it needs (`tab`, `file_path`, ...), taken once right now. - /// - /// It must NOT capture `entity` and call `entity.read(cx)` inside the - /// builder instead: `window.open_dialog`'s builder is re-invoked on every - /// render of *this* view (`ProjectWizard::render` calls - /// `Root::render_dialog_layer`, which calls this builder), and by the - /// time it runs, `self` is already leased for that render — reading it - /// again panics with "cannot read ... while it is already being - /// updated". This isn't timing-dependent (deferring a click doesn't - /// help); it happens on the very next frame the dialog is open, - /// regardless of which button was pressed. - /// - /// Because the builder is a pure function of the snapshot, any click - /// inside it that changes what the dialog should show (tab switch, - /// Browse, Check) must call `refresh_load_config_dialog` afterwards to - /// re-register the dialog with an up-to-date snapshot. - fn open_load_config_dialog(&self, entity: Entity, window: &mut Window, cx: &mut App) { - let tab = self.load_config_tab; - let file_path = self.config_file_path.clone(); - let preview = self.config_preview.clone(); - let error = self.config_error.clone(); - let sheet_link_input = self.sheet_link_input.clone(); - - window.open_dialog(cx, move |dialog, _window, cx| { - let tabs = TabBar::new("load-config-tabs") - .segmented() - .selected_index(match tab { - LoadConfigTab::File => 0, - LoadConfigTab::Sheet => 1, - }) - .on_click({ - let entity = entity.clone(); - move |ix: &usize, window, cx| { - let tab = if *ix == 0 { - LoadConfigTab::File - } else { - LoadConfigTab::Sheet - }; - entity.update(cx, |this, cx| { - this.load_config_tab = tab; - let entity = cx.entity(); - this.refresh_load_config_dialog(entity, window, cx); - }); - } - }) - .child(Tab::new().label("From file")) - .child(Tab::new().label("From Google Sheet")); - - let body = match tab { - LoadConfigTab::File => { - let entity_browse = entity.clone(); - v_flex() - .gap_1() - .child(Label::new("Config file").text_sm()) - .child( - h_flex() - .gap_2() - .child( - div() - .flex_1() - .min_w(px(0.)) - .px_2() - .py_1p5() - .rounded_md() - .border_1() - .border_color(cx.theme().border) - .text_sm() - .text_color(cx.theme().muted_foreground) - .child(if file_path.is_empty() { - "Choose a file…".to_string() - } else { - file_path.clone() - }), - ) - .child( - Button::new("browse-config") - .label("Browse…") - .outline() - .on_click(move |_, window, cx| { - entity_browse.update(cx, |this, cx| { - this.browse_config_file(window, cx) - }); - }), - ), - ) - .into_any_element() - } - LoadConfigTab::Sheet => { - let entity_check = entity.clone(); - v_flex() - .gap_1() - .child(Label::new("Sheet link").text_sm()) - .child( - h_flex() - .gap_2() - .child(Input::new(&sheet_link_input).flex_1()) - .child( - Button::new("check-config-sheet") - .label("Check") - .outline() - .on_click(move |_, window, cx| { - entity_check.update(cx, |this, cx| { - this.load_config_from_sheet(cx); - let entity = cx.entity(); - this.refresh_load_config_dialog(entity, window, cx); - }); - }), - ), - ) - .into_any_element() - } - }; - - let status = match (&preview, &error) { - (Some(p), _) => Some( - inline_message( - "config-status", - format!("Loaded {} columns", p.entries.len()), - MsgKind::Success, - ) - .into_any_element(), - ), - (None, Some(e)) => Some( - inline_message("config-status", e.clone(), MsgKind::Error).into_any_element(), - ), - (None, None) => None, - }; - - let entity_ok = entity.clone(); + }; + let entity = cx.entity(); + window.open_dialog(cx, move |dialog, _, _| { + let (loader_ok, entity) = (loader.clone(), entity.clone()); dialog .title("Load column config") - .child(v_flex().gap_3().child(tabs).child(body).children(status)) + .child(loader.clone()) .button_props( DialogButtonProps::default() .ok_text("Next →") @@ -389,34 +201,25 @@ impl ProjectWizard { .show_cancel(true), ) .on_ok(move |_, _, cx| { - let has_preview = entity_ok.read(cx).config_preview.is_some(); - if has_preview { - entity_ok.update(cx, |this, cx| { - this.column_source = ColumnSource::LoadFromFileOrSheet; - cx.notify(); - }); - true - } else { - false - } + let Some(preview) = loader_ok.read(cx).preview.clone() else { + return false; + }; + entity.update(cx, |this, cx| { + this.config_preview = Some(preview); + this.column_source = ColumnSource::LoadFromFileOrSheet; + this.reset_required_column_defaults(); + cx.notify(); + }); + true }) }); } - /// Re-registers the load-config dialog with a fresh snapshot. Call this - /// after any mutation (inside the dialog) to state the dialog displays — - /// see the note on `open_load_config_dialog` for why the dialog can't - /// just re-read `self` on its own. - fn refresh_load_config_dialog(&self, entity: Entity, window: &mut Window, cx: &mut App) { - window.close_dialog(cx); - self.open_load_config_dialog(entity, window, cx); - } - fn render_advanced_mapping(&mut self, cx: &mut Context) -> impl IntoElement { // Title and File are picked above, in the required-columns section. Listing them here too // showed each one twice with two different answers — the picker's and the config's. let headers: Vec = self - .spreadsheet_headers() + .effective_headers() .into_iter() .filter(|h| { ![self.title_column.as_deref(), self.file_column.as_deref()] @@ -425,7 +228,7 @@ impl ProjectWizard { .any(|chosen| chosen == h) }) .collect(); - let config = self.config_preview.clone(); + let config = self.selected_config().cloned(); let open = self.show_advanced_mapping; Collapsible::new() @@ -461,52 +264,7 @@ impl ProjectWizard { cx.notify(); })), ) - .content({ - let mut table = v_flex().gap_1().mt_2().child( - Label::new("Spreadsheet column → qrate field") - .text_sm() - .text_color(cx.theme().muted_foreground), - ); - if headers.is_empty() { - table = table.child( - Label::new("No spreadsheet columns yet.") - .text_sm() - .text_color(cx.theme().muted_foreground), - ); - } - for header in &headers { - let mapped = config.as_ref().and_then(|c| { - c.entries - .iter() - .find(|e| e.name.eq_ignore_ascii_case(header)) - .map(|e| { - if e.data_type.is_empty() { - e.name.clone() - } else { - format!("{} ({})", e.name, e.data_type) - } - }) - }); - let (target, unmapped) = match mapped { - Some(name) => (name, false), - None => ("Unmapped".to_string(), true), - }; - table = table.child( - h_flex() - .justify_between() - .text_sm() - .py_1() - .border_b_1() - .border_color(cx.theme().border) - // The warning colour already says "unmapped" — a trailing glyph saying it - // again depends on font coverage for nothing. - .when(unmapped, |el| el.text_color(cx.theme().warning)) - .child(header.clone()) - .child(format!("→ {target}")), - ); - } - table - }) + .content(div().mt_2().child(mapping(&headers, config.as_ref(), cx))) } pub(crate) fn render_columns_step( @@ -517,7 +275,7 @@ impl ProjectWizard { let is_blank = self.entry_kind == EntryKind::Blank; let auto_selected = self.column_source == ColumnSource::AutoFromSpreadsheet; let load_selected = self.column_source == ColumnSource::LoadFromFileOrSheet; - let skip_selected = self.column_source == ColumnSource::SkipForNow; + let skip_selected = self.column_source == ColumnSource::DefaultBlank; v_flex() .gap_3() @@ -538,6 +296,7 @@ impl ProjectWizard { ) .on_click(cx.listener(|this, _, _, cx| { this.column_source = ColumnSource::AutoFromSpreadsheet; + this.reset_required_column_defaults(); cx.notify(); })), ) @@ -551,27 +310,29 @@ impl ProjectWizard { cx, ) .on_click(cx.listener(|this, _, window, cx| { - let entity = cx.entity(); - this.open_load_config_dialog(entity, window, cx); + this.open_load_config_dialog(window, cx); })), ) .when(is_blank, |el| { el.child( option_card( "col-skip", - "Skip — set up columns later", - "Start empty; add columns in project settings whenever you're ready.", + "Use default Title and File columns", + "Start with qrate's two required columns. You can add more later.", skip_selected, cx, ) .on_click(cx.listener(|this, _, _, cx| { - this.column_source = ColumnSource::SkipForNow; + this.column_source = ColumnSource::DefaultBlank; + this.reset_required_column_defaults(); cx.notify(); })), ) }) .child(self.render_required_columns(window, cx)) - .child(self.render_advanced_mapping(cx)) + .when(load_selected, |el| { + el.child(self.render_advanced_mapping(cx)) + }) .child( Label::new("You can always adjust this later in project settings.") .text_sm() diff --git a/crates/project-wizard/src/steps/files.rs b/crates/project-wizard/src/steps/files.rs index bd6c2147..cf074935 100644 --- a/crates/project-wizard/src/steps/files.rs +++ b/crates/project-wizard/src/steps/files.rs @@ -1,10 +1,13 @@ use gpui::{prelude::FluentBuilder, *}; use gpui_component::alert::Alert; use gpui_component::button::Button; +use gpui_component::checkbox::Checkbox; use gpui_component::input::Input; use gpui_component::label::Label; use gpui_component::text::Text; -use gpui_component::{ActiveTheme, Disableable, Sizable, StyledExt, h_flex, v_flex}; +use gpui_component::{ActiveTheme, Disableable, Selectable, Sizable, StyledExt, h_flex, v_flex}; + +use file_ingest::duplicates::DuplicatePolicy; use crate::data; use crate::wizard::{EntryKind, ProjectWizard}; @@ -76,7 +79,7 @@ impl ProjectWizard { let _ = window; } - fn set_local_path(&mut self, path: String, _cx: &mut Context) { + pub(crate) fn set_local_path(&mut self, path: String, _cx: &mut Context) { self.local_path = path; match data::load_spreadsheet_preview(&self.local_path) { Ok(preview) => { @@ -91,11 +94,66 @@ impl ProjectWizard { self.revalidate_folder(); } - fn set_folder_path(&mut self, path: String, _cx: &mut Context) { + pub(crate) fn set_folder_path(&mut self, path: String, _cx: &mut Context) { + self.import_paths = vec![std::path::PathBuf::from(&path)]; self.folder_path = path; self.revalidate_folder(); } + pub(crate) fn set_import_paths( + &mut self, + paths: Vec, + _cx: &mut Context, + ) { + if self.entry_kind != EntryKind::Blank { + if paths.len() == 1 && paths[0].is_dir() { + self.set_folder_path(paths[0].to_string_lossy().into_owned(), _cx); + } else { + self.folder_error = Some( + "Multiple files and folders can start a blank project; choose one files folder for a spreadsheet-backed project." + .into(), + ); + } + return; + } + let Ok(mut plan) = file_ingest::plan_paths( + &paths, + &file_ingest::PlanOptions { + recursive: self.recurse_subfolders, + include_root: self.include_root_folder, + ..Default::default() + }, + ) else { + self.folder_error = Some("Those files or folders couldn't be read.".into()); + return; + }; + let folder_path = match paths.as_slice() { + [path] if path.is_dir() => path.to_string_lossy().into_owned(), + _ => { + for component in &mut plan.components { + component.source_path = component.absolute_path.clone(); + } + String::new() + } + }; + let extra_files = plan + .components + .iter() + .filter(|component| component.kind == file_ingest::EntryKind::File) + .map(|component| component.title.clone()) + .collect(); + self.import_paths = paths; + self.folder_path = folder_path; + self.folder_plan = Some(plan); + self.folder_match = Some(data::FolderMatch { + matched_rows: 0, + total_rows: 0, + extra_files, + ambiguous_files: 0, + }); + self.folder_error = None; + } + pub(crate) fn revalidate_folder(&mut self) { if self.folder_path.is_empty() { self.folder_match = None; @@ -117,18 +175,40 @@ impl ProjectWizard { Some(Ok(m)) => { self.folder_match = Some(m); self.folder_error = None; + self.refresh_folder_plan(); } Some(Err(e)) => { self.folder_match = None; + self.folder_plan = None; self.folder_error = Some(e.message().into()); } None => { self.folder_match = None; + self.folder_plan = None; self.folder_error = None; } } } + fn refresh_folder_plan(&mut self) { + self.folder_plan = file_ingest::plan_paths( + &self.import_paths, + &file_ingest::PlanOptions { + recursive: self.recurse_subfolders, + include_root: self.include_root_folder, + ..Default::default() + }, + ) + .ok(); + if self.folder_path.is_empty() + && let Some(plan) = &mut self.folder_plan + { + for component in &mut plan.components { + component.source_path = component.absolute_path.clone(); + } + } + } + pub(crate) fn check_sheet_link(&mut self, auto_advance: bool, cx: &mut Context) { let link = self.sheet_link_input.read(cx).value().to_string(); // Fetch is a blocking network call — run it on a background thread so @@ -186,11 +266,188 @@ impl ProjectWizard { EntryKind::Blank => self.render_blank_files(window, cx).into_any_element(), }; v_flex() + .id("files-drop-area") .gap_3() + .drag_over::(|style, _, _, cx| { + style.bg(cx.theme().secondary_hover) + }) + .on_drop(cx.listener(|this, paths: &ExternalPaths, _, cx| { + let paths = paths.paths(); + if this.entry_kind == EntryKind::Blank { + this.set_import_paths(paths.to_vec(), cx); + } else if paths.len() == 1 && paths[0].is_dir() { + this.set_folder_path(paths[0].to_string_lossy().into_owned(), cx); + } else if this.entry_kind == EntryKind::LocalFile + && paths.len() == 1 + && paths[0].is_file() + { + this.set_local_path(paths[0].to_string_lossy().into_owned(), cx); + } else { + this.folder_error = Some( + "Drop one folder here. You can drop several individual files into the table after the project opens." + .into(), + ); + } + cx.notify(); + })) .child(body) + .when(!self.skip_files, |this| { + this.child( + Checkbox::new("include-root-folder") + .label("Create a row for the selected files folder") + .checked(self.include_root_folder) + .on_click(cx.listener(|this, checked: &bool, _, cx| { + this.include_root_folder = *checked; + this.refresh_folder_plan(); + cx.notify(); + })), + ) + .child( + Label::new( + "Leave this off when the folder only contains this project's material.", + ) + .text_sm() + .text_color(cx.theme().muted_foreground), + ) + .child(self.render_description_profile(cx)) + }) + // Only worth asking about once the folder actually holds a file more than one row + // names; every other import has nothing to decide. + .when( + !self.skip_files + && self + .folder_match + .as_ref() + .is_some_and(|matched| matched.ambiguous_files > 0), + |this| this.child(self.render_duplicate_policy(cx)), + ) .child(self.render_skip_files_toggle(cx)) } + fn render_duplicate_policy(&self, cx: &mut Context) -> impl IntoElement { + let ambiguous = self + .folder_match + .as_ref() + .map_or(0, |matched| matched.ambiguous_files); + let selected = self.duplicate_policy; + v_flex() + .gap_2() + .pt_2() + .border_t_1() + .border_color(cx.theme().border) + .child(Label::new("Files named by more than one row").text_sm()) + .child( + Label::new(format!( + "{ambiguous} file{} match several rows. qrate never picks for you unless you \ + ask it to.", + if ambiguous == 1 { "" } else { "s" } + )) + .text_sm() + .text_color(cx.theme().muted_foreground), + ) + .child( + h_flex().gap_1().flex_wrap().children( + [ + ( + DuplicatePolicy::Skip, + "Leave them for me", + "each file becomes its own row", + ), + ( + DuplicatePolicy::Update, + "Link the first row", + "in spreadsheet order", + ), + (DuplicatePolicy::AddAsNew, "Add as new rows", "link nothing"), + ] + .into_iter() + .enumerate() + .map(|(index, (policy, label, hint))| { + Button::new(("duplicate-policy", index)) + .label(label) + .tooltip(hint) + .outline() + .selected(policy == selected) + .on_click(cx.listener(move |this, _, _, cx| { + this.duplicate_policy = policy; + cx.notify(); + })) + }), + ), + ) + } + + fn render_description_profile(&self, cx: &mut Context) -> impl IntoElement { + let selected = self.description_profile; + v_flex() + .gap_2() + .pt_2() + .border_t_1() + .border_color(cx.theme().border) + .child(Label::new("Archival description standard").text_sm()) + .child( + Label::new( + "Folder and file rows become archival components using this vocabulary.", + ) + .text_sm() + .text_color(cx.theme().muted_foreground), + ) + .child( + h_flex().gap_1().flex_wrap().children( + settings::description::DescriptionProfile::ALL + .into_iter() + .enumerate() + .map(|(index, profile)| { + Button::new(("description-profile", index)) + .label(profile.label()) + .outline() + .selected(profile == selected) + .on_click(cx.listener(move |this, _, window, cx| { + this.description_profile = profile; + let defaults = profile.defaults(); + let folder_label = defaults + .levels + .iter() + .find(|level| level.key == defaults.folder_level_key) + .map(|level| level.label.as_str()) + .unwrap_or("Group"); + let file_label = defaults + .levels + .iter() + .find(|level| level.key == defaults.file_level_key) + .map(|level| level.label.as_str()) + .unwrap_or("Item"); + this.folder_level_input.update(cx, |input, cx| { + input.set_value(folder_label, window, cx) + }); + this.file_level_input.update(cx, |input, cx| { + input.set_value(file_label, window, cx) + }); + cx.notify(); + })) + }), + ), + ) + .child( + h_flex() + .gap_2() + .child( + v_flex() + .flex_1() + .gap_1() + .child(Label::new("Folders are called").text_sm()) + .child(Input::new(&self.folder_level_input)), + ) + .child( + v_flex() + .flex_1() + .gap_1() + .child(Label::new("Files are called").text_sm()) + .child(Input::new(&self.file_level_input)), + ), + ) + } + fn folder_field( &self, browse_id: &'static str, @@ -258,6 +515,7 @@ impl ProjectWizard { _window: &mut Window, cx: &mut Context, ) -> impl IntoElement { + let selected = self.import_paths.len(); v_flex() .gap_3() .child(div().text_lg().font_semibold().child("Add your files")) @@ -267,6 +525,17 @@ impl ProjectWizard { .text_color(cx.theme().muted_foreground), ) .child(self.folder_field("browse-folder-blank", true, cx)) + .when(self.folder_path.is_empty() && selected > 0, |files| { + files.child(inline_message( + "selected-import-paths", + format!( + "Selected {selected} file{} or folder{}.", + if selected == 1 { "" } else { "s" }, + if selected == 1 { "" } else { "s" } + ), + MsgKind::Success, + )) + }) } fn render_local_files( diff --git a/crates/project-wizard/src/steps/review.rs b/crates/project-wizard/src/steps/review.rs index 90e68d8b..2c3c8678 100644 --- a/crates/project-wizard/src/steps/review.rs +++ b/crates/project-wizard/src/steps/review.rs @@ -1,8 +1,14 @@ //! Stage 5 · Review & Create — every path converges here. +use std::collections::HashMap; + +use file_ingest::duplicates::{DuplicatePolicy, ExistingComponent as Existing, Parent, Resolution}; + use gpui::{prelude::FluentBuilder, *}; use gpui_component::description_list::DescriptionList; -use gpui_component::{Sizable, StyledExt, v_flex}; +use gpui_component::label::Label; +use gpui_component::scroll::ScrollableElement; +use gpui_component::{ActiveTheme, Sizable, StyledExt, h_flex, v_flex}; use plugin_api::ColumnMapContributions; use settings::columns::{ColumnSettings, ColumnSettingsMap}; @@ -40,39 +46,8 @@ fn column_settings( continue; }; - let mut settings = ColumnSettings { - authority: entry.authority.clone(), - ..Default::default() - }; - if let Some(on) = entry.spellcheck { - settings.spellcheck = on; - } - if let Some(on) = entry.variant_review { - settings.variant_review = on; - } - // Severity is per producer: the authority names its own, and each plugin's rides in the - // `::Severity` column beside its mapping. - if let (Some(authority), Some(severity)) = - (settings.authority.clone(), entry.authority_severity.clone()) - { - settings.severity.insert(authority, severity); - } - for (plugin, spec) in &maps { - if let Some(cell) = entry.extra.get(&format!("{plugin}::{}", spec.label)) { - let chosen: Vec = cell - .split(';') - .map(str::trim) - .filter(|v| !v.is_empty()) - .map(|v| v.to_string().into()) - .collect(); - ColumnMapContributions::put(plugin, spec, &mut settings, &chosen); - } - if let Some(severity) = entry.extra.get(&format!("{plugin}::Severity")) { - settings - .severity - .insert(plugin.to_string(), severity.to_ascii_lowercase()); - } - } + let mut settings = ColumnSettings::default(); + crate::column_config::apply_entry(entry, &maps, &mut settings); if settings != ColumnSettings::default() { map.insert(header.clone(), settings); @@ -142,23 +117,207 @@ fn project_columns( columns } -fn append_extra_file_rows( - headers: &[String], +fn hierarchy_preview(plan: &file_ingest::ImportPlan, cx: &App) -> AnyElement { + const MAX_VISIBLE: usize = 100; + let files = plan + .components + .iter() + .filter(|component| component.kind == file_ingest::EntryKind::File) + .count(); + let folders = plan.components.len() - files; + let mut depths = Vec::with_capacity(plan.components.len()); + for component in &plan.components { + depths.push(component.parent.map_or(0, |parent| depths[parent] + 1)); + } + + v_flex() + .gap_2() + .child(div().font_semibold().child("Hierarchy preview")) + .child( + Label::new(format!( + "{} component{} · {files} file{} · {folders} folder{}", + plan.components.len(), + if plan.components.len() == 1 { "" } else { "s" }, + if files == 1 { "" } else { "s" }, + if folders == 1 { "" } else { "s" }, + )) + .text_sm() + .text_color(cx.theme().muted_foreground), + ) + .child( + v_flex() + .max_h(px(280.)) + .overflow_y_scrollbar() + .rounded_md() + .border_1() + .border_color(cx.theme().border) + .children( + plan.components + .iter() + .zip(depths) + .take(MAX_VISIBLE) + .enumerate() + .map(|(index, (component, depth))| { + h_flex() + .gap_2() + .px_2() + .py_1() + .pl(px(8. + depth as f32 * 18.)) + .when(index > 0, |row| { + row.border_t_1().border_color(cx.theme().border) + }) + .child( + Label::new(match component.kind { + file_ingest::EntryKind::Directory => "Folder", + file_ingest::EntryKind::File => "File", + }) + .text_xs() + .text_color(cx.theme().muted_foreground), + ) + .child(Label::new(component.title.clone()).text_sm()) + }), + ) + .when(plan.components.len() > MAX_VISIBLE, |tree| { + tree.child( + Label::new(format!( + "… and {} more components", + plan.components.len() - MAX_VISIBLE + )) + .px_2() + .py_1() + .text_sm() + .text_color(cx.theme().muted_foreground), + ) + }), + ) + .when(!plan.warnings.is_empty(), |preview| { + preview.child( + Label::new(format!( + "{} path warning{} will be skipped.", + plan.warnings.len(), + if plan.warnings.len() == 1 { "" } else { "s" } + )) + .text_sm() + .text_color(cx.theme().warning), + ) + }) + .into_any_element() +} + +/// What the Files and Columns steps settled on, as the folder import reads it. +struct FolderImport<'a> { + headers: &'a [String], + title_column: Option<&'a str>, + file_column: Option<&'a str>, + folder: &'a str, + plan: &'a file_ingest::ImportPlan, + description: &'a settings::description::DescriptionConfig, + policy: DuplicatePolicy, +} + +fn append_folder_components( + import: &FolderImport<'_>, rows: &mut Vec>, - file_column: Option<&str>, - folder_match: Option<&crate::data::FolderMatch>, -) { - let Some((files, file_col)) = folder_match - .map(|matched| matched.extra_files.as_slice()) - .zip(file_column.and_then(|name| headers.iter().position(|header| header == name))) +) -> Vec { + let FolderImport { + headers, + title_column, + file_column, + folder, + plan, + description, + policy, + } = *import; + let Some(file_col) = + file_column.and_then(|name| headers.iter().position(|header| header == name)) else { - return; + return Vec::new(); }; - rows.extend(files.iter().map(|file| { - let mut row = vec![String::new(); headers.len()]; - row[file_col] = file.clone(); - row - })); + let title_col = title_column.and_then(|name| headers.iter().position(|header| header == name)); + let mut plan = plan.clone(); + for component in &mut plan.components { + component.level_key = match component.kind { + file_ingest::EntryKind::Directory => &description.folder_level_key, + file_ingest::EntryKind::File => &description.file_level_key, + } + .clone(); + } + for warning in &plan.warnings { + log::warn!("Skipped a path while importing {folder}: {warning:?}"); + } + // The spreadsheet rows are what this import can already be holding, so they are the existing + // components: a file matching one of them links that row instead of adding another. + let existing: Vec = rows + .iter() + .enumerate() + .map(|(source, row)| Existing { + key: source as u64, + absolute_source: None, + filename_keys: row + .get(file_col) + .map(|value| settings::filenames::lookup_keys(value)) + .unwrap_or_default(), + }) + .collect(); + let resolved = + file_ingest::duplicates::resolve(plan, &existing, settings::filenames::keys, policy); + if resolved.ambiguous() > 0 { + log::info!( + "{} file(s) in {folder} are named by more than one row and were left for the archivist", + resolved.ambiguous() + ); + } + let mut next_order: HashMap, i64> = HashMap::new(); + let mut planned_rows = Vec::::with_capacity(resolved.plan.components.len()); + let mut structure: Vec = + Vec::with_capacity(resolved.plan.components.len()); + let parents = resolved.parents.clone(); + let resolutions = resolved.resolutions.clone(); + for (component_index, component) in resolved.plan.components.into_iter().enumerate() { + let source_path = file_ingest::normalized_path(&component.source_path); + let claimed = match &resolutions[component_index] { + Resolution::Skip { existing } | Resolution::Update { existing } => { + Some(*existing as usize) + } + // Several rows name this file. Only "update" picks one, by sheet order. + Resolution::Ambiguous { candidates } => (policy == DuplicatePolicy::Update) + .then(|| candidates.first().map(|row| *row as usize)) + .flatten(), + Resolution::Create => None, + }; + let source = claimed.unwrap_or_else(|| { + let mut row = vec![String::new(); headers.len()]; + if let Some(title_col) = title_col { + row[title_col] = component.title.clone(); + } + if component.kind == file_ingest::EntryKind::File { + row[file_col] = source_path.clone(); + } + rows.push(row); + rows.len() - 1 + }); + planned_rows.push(source); + let parent_id = match parents[component_index] { + Some(Parent::Planned(parent)) => planned_rows.get(parent).copied(), + Some(Parent::Existing(row)) => Some(row as usize), + None => None, + } + .map(|source| source as project::RowId + 1); + let sibling_order = next_order.entry(parent_id).or_default(); + *sibling_order += 1; + structure.push(project::RowStructure { + row_id: source as project::RowId + 1, + parent_id, + level_key: component.level_key, + sibling_order: *sibling_order - 1, + source_path: Some(source_path), + source_kind: Some(match component.kind { + file_ingest::EntryKind::File => project::SourceKind::File, + file_ingest::EntryKind::Directory => project::SourceKind::Directory, + }), + }); + } + structure } impl ProjectWizard { @@ -175,10 +334,10 @@ impl ProjectWizard { LinkMethod::ExactFilename => "exact filename", LinkMethod::CustomPattern => "custom pattern", }); - let spreadsheet_headers = self.spreadsheet_headers(); + let spreadsheet_headers = self.effective_headers(); let columns = project_columns( &spreadsheet_headers, - self.config_preview.as_ref(), + self.selected_config(), self.title_column.as_deref().unwrap_or_default(), self.file_column.as_deref().unwrap_or_default(), ); @@ -192,14 +351,30 @@ impl ProjectWizard { // A file in the folder that no row names is still part of the collection. It arrives as // its own row, empty but for the filename, so it can be catalogued in qrate instead of // being noticed only when someone counts the folder. - if !self.skip_files { - append_extra_file_rows( - &headers, + let description = self.description_config(cx); + let structure = if !self.skip_files { + let Some(plan) = self.folder_plan.as_ref() else { + self.name_error = Some( + "The files preview is no longer available — choose the folder again".into(), + ); + self.step = WizardStep::Files; + return; + }; + append_folder_components( + &FolderImport { + headers: &headers, + title_column: self.title_column.as_deref(), + file_column: self.file_column.as_deref(), + folder: &self.folder_path, + plan, + description: &description, + policy: self.duplicate_policy, + }, &mut rows, - self.file_column.as_deref(), - self.folder_match.as_ref(), - ); - } + ) + } else { + Vec::new() + }; // A project with no rows is a grid with nothing to type into — the first row has to be // created before anything else can be, so create it here rather than making the archivist // find Insert Row on an empty screen. @@ -223,6 +398,46 @@ impl ProjectWizard { }, ) { Ok(file) => { + log::info!( + "created project {file} from {source}: {} rows, {} arranged components, {} profile", + rows.len(), + structure.len(), + description.profile.key() + ); + if !structure.is_empty() + && let Err(error) = + project::write_row_structure(std::path::Path::new(&file), &structure) + { + log::error!("couldn't save the imported folder hierarchy — {error}"); + } + for (key, value) in [ + ( + settings::description::DESCRIPTION_PROFILE_KEY, + description.profile.key().to_string(), + ), + ( + settings::description::DESCRIPTION_LEVELS_KEY, + serde_json::to_string(&description.levels).unwrap_or_default(), + ), + ( + settings::description::FOLDER_LEVEL_KEY, + description.folder_level_key.clone(), + ), + ( + settings::description::FILE_LEVEL_KEY, + description.file_level_key.clone(), + ), + ( + settings::project::IMPORT_DUPLICATE_POLICY_KEY, + self.duplicate_policy.key().to_string(), + ), + ] { + if let Err(error) = + settings::project::write_setting(std::path::Path::new(&file), key, &value) + { + log::error!("couldn't save description profile setting {key} — {error}"); + } + } // Imported notes become Problems-panel entries. Non-fatal: a lost note must not // fail project creation. `open_project` below wakes the diagnostics loader, // which reads them straight back, so this write is the single source of truth. @@ -256,7 +471,7 @@ impl ProjectWizard { // Everything in the column config that `__columns` has no room for. Written before // the project is opened, so the first validation run already sees it. Non-fatal for // the same reason the notes above are. - let settings = column_settings(&headers, self.config_preview.as_ref(), cx); + let settings = column_settings(&headers, self.selected_config(), cx); if !settings.is_empty() { match serde_json::to_string(&settings) { Ok(json) => { @@ -334,23 +549,18 @@ impl ProjectWizard { }) }) .flatten(); - let column_count = self - .config_preview - .as_ref() - .map(|c| c.entries.len()) - .unwrap_or_else(|| self.spreadsheet_headers().len()); + let column_count = self.effective_headers().len(); let columns_line = { - let source_desc = if self.entry_kind == EntryKind::Blank { - "Required Title and File columns" - } else { - match self.column_source { - ColumnSource::AutoFromSpreadsheet => "Auto-matched from spreadsheet", - ColumnSource::LoadFromFileOrSheet => "Loaded from file/Sheet", - ColumnSource::SkipForNow => "Set up later", - } + let source_desc = match self.column_source { + ColumnSource::AutoFromSpreadsheet => "Auto-matched from spreadsheet", + ColumnSource::LoadFromFileOrSheet => "Loaded from file/Sheet", + ColumnSource::DefaultBlank => "Default Title and File columns", }; format!("{source_desc} · {column_count} columns") }; + let folder_plan = (!self.skip_files) + .then_some(self.folder_plan.as_ref()) + .flatten(); v_flex() .gap_3() @@ -376,6 +586,9 @@ impl ProjectWizard { // Blank projects go through Columns too, so always show it. .item("Columns", columns_line, 1), ) + .when_some(folder_plan, |review, plan| { + review.child(hierarchy_preview(plan, cx)) + }) // The shared wizard footer supplies the "Create Project" (Next) and // "← Back" controls — see `ProjectWizard::render_footer`. } @@ -385,9 +598,11 @@ impl ProjectWizard { mod tests { use settings::columns::ColumnType; - use crate::data::{ColumnConfigEntry, ColumnConfigPreview, FolderMatch}; + use crate::data::{ColumnConfigEntry, ColumnConfigPreview}; + + use file_ingest::duplicates::DuplicatePolicy; - use super::{append_extra_file_rows, project_columns}; + use super::{FolderImport, append_folder_components, project_columns}; fn roles(columns: &[crate::project::ProjectColumn]) -> Vec<(&str, ColumnType)> { columns @@ -447,24 +662,108 @@ mod tests { ); } + /// ASNT-77: two rows naming one file is the archivist's ambiguity to settle, not qrate's. #[test] - fn blank_folder_files_become_rows_in_the_file_column() { + fn two_rows_naming_one_file_follow_the_chosen_policy() { + let folder = tempfile::tempdir().unwrap(); + std::fs::write(folder.path().join("one.jpg"), "photo").unwrap(); + let headers: Vec = vec!["Title".into(), "File".into()]; + let description = settings::description::DescriptionProfile::Rad.defaults(); + let plan = file_ingest::plan(folder.path(), &file_ingest::PlanOptions::default()).unwrap(); + let import = |policy| FolderImport { + headers: &headers, + title_column: Some("Title"), + file_column: Some("File"), + folder: folder.path().to_str().unwrap(), + plan: &plan, + description: &description, + policy, + }; + let sheet = || { + vec![ + vec!["A".to_string(), "one.jpg".to_string()], + vec!["B".to_string(), "one.jpg".to_string()], + ] + }; + + // Neither row is picked: the file arrives as its own row, so nothing is claimed wrongly. + let mut rows = sheet(); + let structure = append_folder_components(&import(DuplicatePolicy::Skip), &mut rows); + assert_eq!(rows.len(), 4); + assert_eq!(structure.last().unwrap().row_id, 4); + + // "Link the first row" takes the first in spreadsheet order and adds no row for the file. + let mut rows = sheet(); + let structure = append_folder_components(&import(DuplicatePolicy::Update), &mut rows); + assert_eq!(rows.len(), 3); + assert_eq!(structure.last().unwrap().row_id, 1); + assert_eq!(rows[0], ["A", "one.jpg"]); + } + + #[test] + fn blank_folder_tree_becomes_archival_component_rows() { + let folder = tempfile::tempdir().unwrap(); + std::fs::create_dir(folder.path().join("series")).unwrap(); + std::fs::write(folder.path().join("series").join("one.jpg"), "photo").unwrap(); let headers = vec!["Title".into(), "File".into()]; let mut rows = Vec::new(); - let folder = FolderMatch { - matched_rows: 0, - total_rows: 0, - extra_files: vec!["one.jpg".into(), "two.png".into()], - }; + let description = settings::description::DescriptionProfile::Rad.defaults(); + let plan = file_ingest::plan(folder.path(), &file_ingest::PlanOptions::default()).unwrap(); - append_extra_file_rows(&headers, &mut rows, Some("File"), Some(&folder)); + let structure = append_folder_components( + &FolderImport { + headers: &headers, + title_column: Some("Title"), + file_column: Some("File"), + folder: folder.path().to_str().unwrap(), + plan: &plan, + description: &description, + policy: DuplicatePolicy::Skip, + }, + &mut rows, + ); - assert_eq!( - rows, - [ - vec![String::new(), "one.jpg".into()], - vec![String::new(), "two.png".into()], - ] + assert_eq!(rows.len(), 3); + assert_eq!(rows[1], ["series", ""]); + assert_eq!(rows[2], ["one.jpg", "series/one.jpg"]); + assert_eq!(structure[1].parent_id, Some(1)); + assert_eq!(structure[2].parent_id, Some(2)); + assert_eq!(structure[1].level_key, "series"); + assert_eq!(structure[2].level_key, "item"); + } + + #[test] + fn excluding_the_selected_root_starts_with_its_children() { + let folder = tempfile::tempdir().unwrap(); + std::fs::create_dir(folder.path().join("series")).unwrap(); + std::fs::write(folder.path().join("series").join("one.jpg"), "photo").unwrap(); + let headers = vec!["Title".into(), "File".into()]; + let description = settings::description::DescriptionProfile::Rad.defaults(); + let plan = file_ingest::plan( + folder.path(), + &file_ingest::PlanOptions { + include_root: false, + ..Default::default() + }, + ) + .unwrap(); + let mut rows = Vec::new(); + + let structure = append_folder_components( + &FolderImport { + headers: &headers, + title_column: Some("Title"), + file_column: Some("File"), + folder: folder.path().to_str().unwrap(), + plan: &plan, + description: &description, + policy: DuplicatePolicy::Skip, + }, + &mut rows, ); + + assert_eq!(rows, [["series", ""], ["one.jpg", "series/one.jpg"]]); + assert_eq!(structure[0].parent_id, None); + assert_eq!(structure[1].parent_id, Some(1)); } } diff --git a/crates/project-wizard/src/wizard.rs b/crates/project-wizard/src/wizard.rs index 23df8599..83ef5f25 100644 --- a/crates/project-wizard/src/wizard.rs +++ b/crates/project-wizard/src/wizard.rs @@ -48,13 +48,7 @@ pub enum ColumnSource { AutoFromSpreadsheet, LoadFromFileOrSheet, /// Blank projects have no spreadsheet to derive columns from — defer setup. - SkipForNow, -} - -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub enum LoadConfigTab { - File, - Sheet, + DefaultBlank, } #[derive(Clone, PartialEq)] @@ -93,10 +87,17 @@ pub struct ProjectWizard { pub(crate) spreadsheet_preview: Option, pub(crate) local_error: Option, pub(crate) folder_path: String, + pub(crate) import_paths: Vec, pub(crate) folder_match: Option, + /// The exact hierarchy shown on Review and consumed by project creation. + pub(crate) folder_plan: Option, pub(crate) folder_error: Option, /// "I'll add files later" — skips folder matching and the whole Link step. pub(crate) skip_files: bool, + pub(crate) description_profile: settings::description::DescriptionProfile, + pub(crate) duplicate_policy: file_ingest::duplicates::DuplicatePolicy, + pub(crate) folder_level_input: Entity, + pub(crate) file_level_input: Entity, // Files step - Sheet pub(crate) sheet_link_input: Entity, @@ -109,14 +110,15 @@ pub struct ProjectWizard { pub(crate) show_advanced_pattern: bool, /// Look for filenames in subfolders too, not just directly in the files folder. pub(crate) recurse_subfolders: bool, + /// Whether the selected files root is itself an archival component. + pub(crate) include_root_folder: bool, // Columns step pub(crate) column_source: ColumnSource, pub(crate) show_advanced_mapping: bool, - pub(crate) load_config_tab: LoadConfigTab, - pub(crate) config_file_path: String, + /// Kept once opened, so reopening the dialog shows what was last loaded. + pub(crate) config_loader: Option>, pub(crate) config_preview: Option, - pub(crate) config_error: Option, pub(crate) title_column: Option, pub(crate) file_column: Option, pub(crate) title_picker: Entity>>, @@ -140,6 +142,36 @@ fn required_columns(title: Option<&str>, file: Option<&str>) -> Result<(), &'sta } Ok(()) } + +fn steps_for(entry_kind: EntryKind, skip_files: bool) -> Vec<(&'static str, WizardStep)> { + let mut items = vec![("Name", WizardStep::Name), ("Files", WizardStep::Files)]; + if !(skip_files || entry_kind == EntryKind::Blank) { + items.push(("Link", WizardStep::Link)); + } + items.push(("Columns", WizardStep::Columns)); + items.push(("Create", WizardStep::Review)); + items +} + +fn effective_headers_for( + entry_kind: EntryKind, + column_source: ColumnSource, + spreadsheet_headers: Vec, + config: Option<&ColumnConfigPreview>, +) -> Vec { + if entry_kind == EntryKind::Blank && column_source == ColumnSource::LoadFromFileOrSheet { + return config + .map(|preview| { + preview + .entries + .iter() + .map(|entry| entry.name.clone()) + .collect() + }) + .unwrap_or_default(); + } + spreadsheet_headers +} impl ProjectWizard { pub fn new(entry_kind: EntryKind, window: &mut Window, cx: &mut Context) -> Self { window.set_window_title("New Project — qrate"); @@ -157,6 +189,16 @@ impl ProjectWizard { .new(|cx| InputState::new(window, cx).placeholder("docs.google.com/spreadsheets/d/…")); let link_pattern_input = cx.new(|cx| InputState::new(window, cx).placeholder("e.g. {id}_*.jpg")); + let folder_level_input = cx.new(|cx| { + InputState::new(window, cx) + .placeholder("Folder level") + .default_value("Series") + }); + let file_level_input = cx.new(|cx| { + InputState::new(window, cx) + .placeholder("File level") + .default_value("Item") + }); let default_save_dir = dirs::document_dir() .map(|d| d.join("qrate")) @@ -231,9 +273,15 @@ impl ProjectWizard { spreadsheet_preview: None, local_error: None, folder_path: String::new(), + import_paths: Vec::new(), folder_match: None, + folder_plan: None, folder_error: None, skip_files: false, + description_profile: settings::description::DescriptionProfile::Rad, + duplicate_policy: file_ingest::duplicates::DuplicatePolicy::default(), + folder_level_input, + file_level_input, sheet_link_input, sheet_check: None, sheet_error: None, @@ -243,17 +291,16 @@ impl ProjectWizard { // On by default: `table::photos` resolves rows against the whole tree, so an // off-by-default check would report fewer matches than the app will actually find. recurse_subfolders: true, + include_root_folder: false, // Blank has no spreadsheet to auto-derive columns from. column_source: if entry_kind == EntryKind::Blank { - ColumnSource::SkipForNow + ColumnSource::DefaultBlank } else { ColumnSource::AutoFromSpreadsheet }, show_advanced_mapping: false, - load_config_tab: LoadConfigTab::File, - config_file_path: String::new(), + config_loader: None, config_preview: None, - config_error: None, title_column: (entry_kind == EntryKind::Blank).then(|| "Title".to_string()), file_column: (entry_kind == EntryKind::Blank).then(|| "File".to_string()), title_picker, @@ -268,6 +315,29 @@ impl ProjectWizard { self.name_input.read(cx).value().to_string() } + pub(crate) fn description_config(&self, cx: &App) -> settings::description::DescriptionConfig { + let mut config = self.description_profile.defaults(); + let folder_label = self.folder_level_input.read(cx).value().trim().to_string(); + let file_label = self.file_level_input.read(cx).value().trim().to_string(); + if let Some(level) = config + .levels + .iter_mut() + .find(|level| level.key == config.folder_level_key) + && !folder_label.is_empty() + { + level.label = folder_label; + } + if let Some(level) = config + .levels + .iter_mut() + .find(|level| level.key == config.file_level_key) + && !file_label.is_empty() + { + level.label = file_label; + } + config + } + pub(crate) fn spreadsheet_headers(&self) -> Vec { match self.entry_kind { // Sheet reuses `spreadsheet_preview` too — its fetched CSV is parsed the @@ -281,6 +351,22 @@ impl ProjectWizard { } } + /// Headers that the project being created will actually contain. + pub(crate) fn effective_headers(&self) -> Vec { + effective_headers_for( + self.entry_kind, + self.column_source, + self.spreadsheet_headers(), + self.config_preview.as_ref(), + ) + } + + pub(crate) fn selected_config(&self) -> Option<&ColumnConfigPreview> { + (self.column_source == ColumnSource::LoadFromFileOrSheet) + .then_some(self.config_preview.as_ref()) + .flatten() + } + /// The Link step is skipped when there's no folder to link against /// (files skipped) or no spreadsheet rows to link (Blank). pub(crate) fn skips_link(&self) -> bool { @@ -371,9 +457,7 @@ impl ProjectWizard { /// Shared by the manual Next click and the auto-advance after a /// just-succeeded sheet check (see `steps/files.rs::check_sheet_link`). pub(crate) fn advance_past_files(&mut self) { - self.step = if self.entry_kind == EntryKind::Blank { - WizardStep::Review - } else if self.skips_link() { + self.step = if self.skips_link() { WizardStep::Columns } else { WizardStep::Link @@ -381,15 +465,7 @@ impl ProjectWizard { } fn breadcrumb_items(&self) -> Vec<(&'static str, WizardStep)> { - let mut items = vec![("Name", WizardStep::Name), ("Files", WizardStep::Files)]; - if !self.skips_link() { - items.push(("Link", WizardStep::Link)); - } - if self.entry_kind != EntryKind::Blank { - items.push(("Columns", WizardStep::Columns)); - } - items.push(("Create", WizardStep::Review)); - items + steps_for(self.entry_kind, self.skip_files) } fn step_index(&self) -> usize { @@ -437,7 +513,6 @@ impl ProjectWizard { WizardStep::Link } } - (EntryKind::Blank, WizardStep::Review) => WizardStep::Files, (_, WizardStep::Review) => WizardStep::Columns, }; cx.notify(); @@ -569,6 +644,29 @@ impl Render for ProjectWizard { } pub fn open_project_wizard(entry_kind: EntryKind, cx: &mut App) { + open_project_wizard_seeded(entry_kind, None, None, cx); +} + +pub(crate) fn open_project_wizard_seeded( + entry_kind: EntryKind, + spreadsheet: Option, + folder: Option, + cx: &mut App, +) { + open_project_wizard_seeded_paths( + entry_kind, + spreadsheet, + folder.map(std::path::PathBuf::from).into_iter().collect(), + cx, + ); +} + +pub(crate) fn open_project_wizard_seeded_paths( + entry_kind: EntryKind, + spreadsheet: Option, + paths: Vec, + cx: &mut App, +) { let bounds = Bounds::centered(None, size(px(560.0), px(680.0)), cx); let window_options = WindowOptions { titlebar: Some(TitleBar::title_bar_options()), @@ -580,7 +678,16 @@ pub fn open_project_wizard(entry_kind: EntryKind, cx: &mut App) { // Open synchronously: gpui quits when the window list is empty (non-macOS), so a window // spawned from an async task would leave a zero-window gap that kills the app mid-transition. if let Ok(window_handle) = cx.open_window(window_options, |window, cx| { - let view = cx.new(|cx| ProjectWizard::new(entry_kind, window, cx)); + let view = cx.new(|cx| { + let mut wizard = ProjectWizard::new(entry_kind, window, cx); + if let Some(path) = spreadsheet { + wizard.set_local_path(path, cx); + } + if !paths.is_empty() { + wizard.set_import_paths(paths, cx); + } + wizard + }); cx.new(|cx| Root::new(view, window, cx)) }) { WindowRegistry::register(WIZARD_WINDOW_KIND, window_handle.into(), cx); @@ -638,7 +745,10 @@ pub(crate) fn option_card( #[cfg(test)] mod tests { - use crate::wizard::required_columns; + use crate::data::{ColumnConfigEntry, ColumnConfigPreview}; + use crate::wizard::{ + ColumnSource, EntryKind, WizardStep, effective_headers_for, required_columns, steps_for, + }; #[test] fn columns_step_requires_two_distinct_roles() { @@ -647,4 +757,46 @@ mod tests { assert!(required_columns(Some("Title"), None).is_err()); assert!(required_columns(Some("Same"), Some("Same")).is_err()); } + + #[test] + fn blank_projects_visit_columns_before_review() { + let steps: Vec<_> = steps_for(EntryKind::Blank, false) + .into_iter() + .map(|(_, step)| step) + .collect(); + assert_eq!( + steps, + vec![ + WizardStep::Name, + WizardStep::Files, + WizardStep::Columns, + WizardStep::Review + ] + ); + } + + #[test] + fn blank_config_entries_become_dataset_headers() { + let config = ColumnConfigPreview { + entries: vec![ + ColumnConfigEntry { + name: "Object Name".into(), + ..Default::default() + }, + ColumnConfigEntry { + name: "Digital Path".into(), + ..Default::default() + }, + ], + }; + assert_eq!( + effective_headers_for( + EntryKind::Blank, + ColumnSource::LoadFromFileOrSheet, + vec!["Title".into(), "File".into()], + Some(&config), + ), + ["Object Name", "Digital Path"] + ); + } } diff --git a/crates/settings/src/columns.rs b/crates/settings/src/columns.rs index 6adc4057..bd474d3f 100644 --- a/crates/settings/src/columns.rs +++ b/crates/settings/src/columns.rs @@ -43,6 +43,12 @@ pub enum ColumnType { Url, /// An accession number, call number, or other opaque handle. Identifier, + /// The archival level assigned to this component. + DescriptionLevel, + /// A stable reference to this component's broader parent. + ParentComponent, + /// A file or directory path retained from filesystem ingest. + SourcePath, } impl ColumnType { @@ -56,11 +62,14 @@ impl ColumnType { ColumnType::Number => "Number", ColumnType::Url => "Url", ColumnType::Identifier => "Identifier", + ColumnType::DescriptionLevel => "Description Level", + ColumnType::ParentComponent => "Parent Component", + ColumnType::SourcePath => "Source Path", } } /// Every type, in the order the wizard offers them. - pub const ALL: [ColumnType; 7] = [ + pub const ALL: [ColumnType; 10] = [ ColumnType::Title, ColumnType::Text, ColumnType::Date, @@ -68,6 +77,9 @@ impl ColumnType { ColumnType::Number, ColumnType::Url, ColumnType::Identifier, + ColumnType::DescriptionLevel, + ColumnType::ParentComponent, + ColumnType::SourcePath, ]; /// Read a declared type. Case- and space-insensitive, and it accepts the spellings a person @@ -81,6 +93,11 @@ impl ColumnType { "number" | "integer" | "int" | "float" | "decimal" => ColumnType::Number, "url" | "uri" | "link" => ColumnType::Url, "identifier" | "id" => ColumnType::Identifier, + "descriptionlevel" | "description level" | "levelofdescription" => { + ColumnType::DescriptionLevel + } + "parentcomponent" | "parent component" | "parent" => ColumnType::ParentComponent, + "sourcepath" | "source path" => ColumnType::SourcePath, _ => ColumnType::Text, } } diff --git a/crates/settings/src/description.rs b/crates/settings/src/description.rs new file mode 100644 index 00000000..c0e9e553 --- /dev/null +++ b/crates/settings/src/description.rs @@ -0,0 +1,217 @@ +//! Archival description vocabularies used by folder import and hierarchy editing. + +use serde::{Deserialize, Serialize}; + +pub const DESCRIPTION_PROFILE_KEY: &str = "description_profile"; +pub const DESCRIPTION_LEVELS_KEY: &str = "description_levels"; +pub const FOLDER_LEVEL_KEY: &str = "folder_level_key"; +pub const FILE_LEVEL_KEY: &str = "file_level_key"; +pub const HIERARCHY_EXPANDED_KEY: &str = "table_hierarchy_expanded"; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum DescriptionProfile { + #[default] + Rad, + Dacs, + Isadg, + Ric, + Custom, +} + +impl DescriptionProfile { + pub const ALL: [Self; 5] = [Self::Rad, Self::Dacs, Self::Isadg, Self::Ric, Self::Custom]; + + pub fn key(self) -> &'static str { + match self { + Self::Rad => "rad", + Self::Dacs => "dacs", + Self::Isadg => "isadg", + Self::Ric => "ric", + Self::Custom => "custom", + } + } + + pub fn label(self) -> &'static str { + match self { + Self::Rad => "RAD", + Self::Dacs => "DACS", + Self::Isadg => "ISAD(G)", + Self::Ric => "Records in Contexts", + Self::Custom => "Custom", + } + } + + pub fn parse(value: &str) -> Self { + match value.trim().to_ascii_lowercase().as_str() { + "dacs" => Self::Dacs, + "isadg" | "isad(g)" => Self::Isadg, + "ric" => Self::Ric, + "custom" => Self::Custom, + _ => Self::Rad, + } + } + + pub fn defaults(self) -> DescriptionConfig { + let names: &[(&str, &str)] = match self { + Self::Rad => &[ + ("fonds", "Fonds"), + ("collection", "Collection"), + ("sous_fonds", "Sous-fonds"), + ("series", "Series"), + ("subseries", "Subseries"), + ("file", "File"), + ("item", "Item"), + ], + Self::Dacs => &[ + ("collection", "Collection"), + ("record_group", "Record group"), + ("series", "Series"), + ("subseries", "Subseries"), + ("file", "File"), + ("item", "Item"), + ], + Self::Isadg => &[ + ("fonds", "Fonds"), + ("subfonds", "Sub-fonds"), + ("series", "Series"), + ("subseries", "Sub-series"), + ("file", "File"), + ("item", "Item"), + ], + Self::Ric => &[("record_set", "Record set"), ("record", "Record")], + Self::Custom => &[("group", "Group"), ("item", "Item")], + }; + let levels = names + .iter() + .enumerate() + .map(|(index, (key, label))| DescriptionLevel { + key: (*key).into(), + label: (*label).into(), + broader_default: index.checked_sub(1).map(|index| names[index].0.into()), + }) + .collect(); + let (folder_level_key, file_level_key) = match self { + Self::Ric => ("record_set", "record"), + Self::Custom => ("group", "item"), + _ => ("series", "item"), + }; + DescriptionConfig { + profile: self, + levels, + folder_level_key: folder_level_key.into(), + file_level_key: file_level_key.into(), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DescriptionLevel { + pub key: String, + pub label: String, + #[serde(default)] + pub broader_default: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DescriptionConfig { + pub profile: DescriptionProfile, + pub levels: Vec, + pub folder_level_key: String, + pub file_level_key: String, +} + +impl DescriptionConfig { + pub fn from_values(values: &std::collections::HashMap) -> Self { + let profile = values + .get(DESCRIPTION_PROFILE_KEY) + .map(|value| DescriptionProfile::parse(&value.text())) + .unwrap_or_default(); + let mut config = profile.defaults(); + if let Some(levels) = values + .get(DESCRIPTION_LEVELS_KEY) + .and_then(|value| serde_json::from_str(&value.text()).ok()) + { + config.levels = levels; + } + if let Some(value) = values.get(FOLDER_LEVEL_KEY) { + config.folder_level_key = value.text().to_string(); + } + if let Some(value) = values.get(FILE_LEVEL_KEY) { + config.file_level_key = value.text().to_string(); + } + config + } + + pub fn validate(&self) -> Result<(), &'static str> { + if self.levels.is_empty() { + return Err("a description vocabulary needs at least one level"); + } + let mut keys = std::collections::HashSet::new(); + for level in &self.levels { + if level.key.trim().is_empty() || level.label.trim().is_empty() { + return Err("description level keys and labels cannot be empty"); + } + if !keys.insert(level.key.as_str()) { + return Err("description level keys must be unique"); + } + } + if !keys.contains(self.folder_level_key.as_str()) + || !keys.contains(self.file_level_key.as_str()) + { + return Err("folder and file defaults must name configured levels"); + } + if self.levels.iter().any(|level| { + level + .broader_default + .as_ref() + .is_some_and(|key| !keys.contains(key.as_str())) + }) { + return Err("broader defaults must name configured levels"); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_standard_profile_has_valid_editable_defaults() { + for profile in DescriptionProfile::ALL { + let config = profile.defaults(); + assert!(config.validate().is_ok(), "{}", profile.label()); + assert!(!config.levels.is_empty()); + } + } + + #[test] + fn project_values_override_vocabulary_and_import_defaults() { + let mut values = std::collections::HashMap::new(); + values.insert( + DESCRIPTION_PROFILE_KEY.into(), + crate::Val::Text("custom".into()), + ); + values.insert( + DESCRIPTION_LEVELS_KEY.into(), + crate::Val::Text( + serde_json::json!([ + {"key": "box", "label": "Box"}, + {"key": "folder", "label": "Folder", "broader_default": "box"} + ]) + .to_string() + .into(), + ), + ); + values.insert(FOLDER_LEVEL_KEY.into(), crate::Val::Text("box".into())); + values.insert(FILE_LEVEL_KEY.into(), crate::Val::Text("folder".into())); + + let config = DescriptionConfig::from_values(&values); + assert_eq!(config.profile, DescriptionProfile::Custom); + assert_eq!(config.levels[0].label, "Box"); + assert_eq!(config.folder_level_key, "box"); + assert_eq!(config.file_level_key, "folder"); + assert!(config.validate().is_ok()); + } +} diff --git a/crates/settings/src/history.rs b/crates/settings/src/history.rs index ee62281f..5e1342dd 100644 --- a/crates/settings/src/history.rs +++ b/crates/settings/src/history.rs @@ -614,6 +614,7 @@ mod tests { &headers, &[1], &[vec!["One.".into()]], + None, &[typed.clone(), fixed.clone()], ) .unwrap(); @@ -655,7 +656,7 @@ mod tests { ) }) .collect(); - save_dataset(&path, &headers, &[1], &[vec!["5".into()]], &entries).unwrap(); + save_dataset(&path, &headers, &[1], &[vec!["5".into()]], None, &entries).unwrap(); let first = page(&path, EntryId::MAX, 2, None).unwrap(); let (day, time) = local_times(&[first[0].entry.at]).remove(0); @@ -712,7 +713,7 @@ mod tests { None, ), ]; - save_dataset(&path, &headers, &[1], &[vec!["b".into()]], &entries).unwrap(); + save_dataset(&path, &headers, &[1], &[vec!["b".into()]], None, &entries).unwrap(); let row = page(&path, EntryId::MAX, 10, Some(1)).unwrap(); assert_eq!(row.iter().map(|l| l.entry.id).collect::>(), vec![1]); @@ -771,7 +772,7 @@ mod tests { ) }) .collect(); - save_dataset(&path, &headers, &[1], &[vec!["6".into()]], &log).unwrap(); + save_dataset(&path, &headers, &[1], &[vec!["6".into()]], None, &log).unwrap(); // The oldest entry of the six, named — the one a retention rule would otherwise reach first. set_label(&path, 1, Some("Before ingest")).unwrap(); @@ -820,6 +821,7 @@ mod tests { &headers, &[1], &[vec!["f".into()]], + None, &[first.clone(), second.clone()], ) .unwrap(); diff --git a/crates/settings/src/lib.rs b/crates/settings/src/lib.rs index fc51ebec..b951b10e 100644 --- a/crates/settings/src/lib.rs +++ b/crates/settings/src/lib.rs @@ -2,6 +2,7 @@ //! settings-window shell (`SettingsWindow`). Product-specific pages live in `app`. pub mod columns; +pub mod description; pub mod dirty; pub mod filenames; pub mod history; diff --git a/crates/settings/src/path_picker.rs b/crates/settings/src/path_picker.rs index 87d49764..ac5b05c9 100644 --- a/crates/settings/src/path_picker.rs +++ b/crates/settings/src/path_picker.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use gpui::*; use gpui_component::{ - IconName, Sizable, Size, + ActiveTheme, IconName, Sizable, Size, button::Button, h_flex, input::{Input, InputState}, @@ -45,12 +45,22 @@ impl RenderOnce for PathPickerApp { }; let on_pick = Arc::clone(&self.on_pick); + let on_drop = Arc::clone(&self.on_pick); let field_size = self.field_size; h_flex() .gap_2() .w_full() + .drag_over::(|style, _, _, cx| style.bg(cx.theme().secondary_hover)) + .on_drop(move |paths: &ExternalPaths, _, cx| { + let [path] = paths.paths() else { + return; + }; + if (files && path.is_file()) || (directories && path.is_dir()) { + on_drop(path.to_string_lossy().into_owned().into(), cx); + } + }) .child( div().flex_1().min_w(px(0.)).child( Input::new(&self.input) diff --git a/crates/settings/src/project.rs b/crates/settings/src/project.rs index b664b511..00c0c800 100644 --- a/crates/settings/src/project.rs +++ b/crates/settings/src/project.rs @@ -1,4 +1,4 @@ -//! On-disk `.qrate` project file (v3). One SQLite database per project, +//! On-disk `.qrate` project file (v4). One SQLite database per project, //! written once by the "New Project" wizard. Runs in `journal_mode=DELETE`, //! not WAL: with a single short-lived connection per operation WAL buys no //! concurrency, and its `-wal`/`-shm` siblings only vanish when the *last* @@ -24,7 +24,7 @@ //! get `__notes` created lazily on first write rather than a migration pass, so //! opening a project stays a read-only operation. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, mpsc}; use std::time::Duration; @@ -36,18 +36,68 @@ use rusqlite::{Connection, OptionalExtension as _, params}; /// `PRAGMA application_id` value so `file`-style tools can recognize `.qrate`. const QRATE_APPLICATION_ID: i32 = 1097887558; -const QRATE_SCHEMA_VERSION: i32 = 3; +const QRATE_SCHEMA_VERSION: i32 = 4; /// Private identity of one `dataset_main` row. Unlike the source index shown in the `#` column, /// this value survives inserts, deletes, and save/reload cycles. pub type RowId = i64; type LoadedDataset = (Vec, Vec, Vec>); +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SourceKind { + File, + Directory, +} + +impl SourceKind { + fn as_str(self) -> &'static str { + match self { + Self::File => "file", + Self::Directory => "directory", + } + } + + fn parse(value: &str) -> Option { + match value { + "file" => Some(Self::File), + "directory" => Some(Self::Directory), + _ => None, + } + } +} + +/// Private arrangement metadata for one archival component. None of these values becomes a +/// visible dataset column unless the user explicitly maps it during export. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RowStructure { + pub row_id: RowId, + pub parent_id: Option, + pub level_key: String, + pub sibling_order: i64, + pub source_path: Option, + pub source_kind: Option, +} + +const ROW_STRUCTURE_DDL: &str = r#" + CREATE TABLE IF NOT EXISTS __row_structure ( + row_id INTEGER PRIMARY KEY, + parent_id INTEGER REFERENCES dataset_main(_row_id) ON DELETE CASCADE, + level_key TEXT NOT NULL, + sibling_order INTEGER NOT NULL, + source_path TEXT, + source_kind TEXT CHECK (source_kind IN ('file', 'directory')), + FOREIGN KEY(row_id) REFERENCES dataset_main(_row_id) ON DELETE CASCADE + ); +"#; + /// `__settings` key for the files folder chosen in the wizard's Files step. Only the path is /// kept — qrate never copies source files, so the table crate re-resolves row images against /// this folder every time the project opens (see `table::photos`). pub const FILES_FOLDER_KEY: &str = "files_folder"; +/// What an import does with material the project already holds: skip, update, or add_as_new. +pub const IMPORT_DUPLICATE_POLICY_KEY: &str = "import_duplicate_policy"; + /// `__settings` key for the spreadsheet this project pushes to. The id alone, not the URL — the /// link is derivable (`data_exchange::google::sheet_url`) and the id is what the Sheets API takes. /// Written when the user picks a sheet through Google's chooser, which is also what grants qrate @@ -139,6 +189,25 @@ impl CurrentProject { queue_write(&file, key, &value, cx); } + /// Follow the grid's column set. The Settings pages, the declared types and the preview + /// resolver all read this list, so a column added or renamed in the grid stays invisible to + /// them until it lands here. + pub fn set_columns(headers: Vec, renamed: Option<(&str, &str)>, cx: &mut gpui::App) { + // Undo lands here after every step, and a write wakes every observer of the project. + if cx + .try_global::() + .is_none_or(|project| renamed.is_none() && project.data.headers == headers) + { + return; + } + let project = cx.global_mut::(); + if let Some((before, after)) = renamed + && let Some(column) = project.data.columns.iter_mut().find(|c| c.name == before) + { + column.name = after.to_string(); + } + project.data.headers = headers; + } /// Declares a column's type, updating the cache validators read and writing `__columns`. /// Synchronous rather than queued: [`ProjectSettingsWriter`] is keyed by `__settings` key and /// this is another table, and picking a type is one deliberate click, not a drag. @@ -252,7 +321,7 @@ pub struct ProjectSpec<'a> { pub rows: &'a [Vec], } -/// Creates `path` (a `.qrate` file) with the v3 schema and the imported data. +/// Creates `path` (a `.qrate` file) with the current schema and imported data. pub fn create_project_file(path: &Path, spec: &ProjectSpec<'_>) -> Result<()> { let &ProjectSpec { name, @@ -323,6 +392,8 @@ pub fn create_project_file(path: &Path, spec: &ProjectSpec<'_>) -> Result<()> { if !headers.is_empty() { write_dataset(&conn, headers, rows)?; } + conn.execute_batch(ROW_STRUCTURE_DDL) + .context("Create __row_structure")?; Ok(()) } @@ -419,6 +490,103 @@ fn load_dataset(conn: &Connection) -> Result { Ok((headers, row_ids, rows)) } +/// Reads the optional hierarchy without upgrading an older project. Rows missing from this table +/// are ungrouped roots, so v3 and partially arranged projects remain usable. +pub fn read_row_structure(path: &Path) -> Result> { + let conn = open_ro(path)?; + if !table_exists(&conn, "__row_structure")? { + return Ok(Vec::new()); + } + read_row_structure_from(&conn) +} + +fn read_row_structure_from(conn: &Connection) -> Result> { + let mut stmt = conn.prepare( + "SELECT row_id, parent_id, level_key, sibling_order, source_path, source_kind + FROM __row_structure ORDER BY parent_id, sibling_order, row_id", + )?; + let rows = stmt.query_map([], |row| { + let source_kind: Option = row.get(5)?; + Ok(RowStructure { + row_id: row.get(0)?, + parent_id: row.get(1)?, + level_key: row.get(2)?, + sibling_order: row.get(3)?, + source_path: row.get(4)?, + source_kind: source_kind.as_deref().and_then(SourceKind::parse), + }) + })?; + rows.collect::>>() + .context("Read row structure") +} + +fn table_exists(conn: &Connection, name: &str) -> Result { + Ok(conn.query_row( + "SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name = ?1", + params![name], + |row| row.get::<_, i64>(0), + )? != 0) +} + +/// Replaces the hierarchy in one transaction. This is the lazy v3-to-v4 migration point: opening +/// an older project stays read-only, while its first structural edit creates the private table. +pub fn write_row_structure(path: &Path, structure: &[RowStructure]) -> Result<()> { + validate_row_structure(structure)?; + let mut conn = open_rw(path)?; + conn.pragma_update(None, "foreign_keys", "ON")?; + let tx = conn.transaction().context("Begin row structure update")?; + tx.execute_batch(ROW_STRUCTURE_DDL) + .context("Create __row_structure")?; + tx.execute("DELETE FROM __row_structure", []) + .context("Clear row structure")?; + insert_row_structure(&tx, structure)?; + tx.pragma_update(None, "user_version", QRATE_SCHEMA_VERSION) + .context("Set user_version")?; + tx.commit().context("Commit row structure update") +} + +fn insert_row_structure(conn: &Connection, structure: &[RowStructure]) -> Result<()> { + let mut stmt = conn.prepare( + "INSERT INTO __row_structure + (row_id, parent_id, level_key, sibling_order, source_path, source_kind) + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + )?; + for component in structure { + stmt.execute(params![ + component.row_id, + component.parent_id, + component.level_key, + component.sibling_order, + component.source_path, + component.source_kind.map(SourceKind::as_str), + ]) + .with_context(|| format!("Insert structure for row {}", component.row_id))?; + } + Ok(()) +} + +fn validate_row_structure(structure: &[RowStructure]) -> Result<()> { + let by_id: HashMap<_, _> = structure.iter().map(|row| (row.row_id, row)).collect(); + if by_id.len() != structure.len() { + anyhow::bail!("row structure contains duplicate row ids"); + } + for row in structure { + if row.level_key.trim().is_empty() { + anyhow::bail!("row {} has an empty description level", row.row_id); + } + let mut parent = row.parent_id; + let mut remaining = structure.len(); + while let Some(parent_id) = parent { + if parent_id == row.row_id || remaining == 0 { + anyhow::bail!("row structure contains a cycle at row {}", row.row_id); + } + parent = by_id.get(&parent_id).and_then(|parent| parent.parent_id); + remaining -= 1; + } + } + Ok(()) +} + /// Reads one `__settings` value from a `.qrate` file (e.g. the dock layout). pub fn read_setting(path: &Path, key: &str) -> Result> { let conn = open_ro(path)?; @@ -934,24 +1102,82 @@ fn write_dataset(conn: &Connection, headers: &[String], rows: &[Vec]) -> /// the connection drops). `headers`/`rows` must be in the project's *original* column order: the /// `.qrate` schema keeps a fixed physical column order, and the separately-saved column layout /// maps that to display order. A blank project (no headers, no `dataset_main`) is a no-op. +/// +/// `structure` is the live hierarchy, written in the same transaction as the rows it names; +/// `None` keeps whatever hierarchy the file already holds. pub fn save_dataset( path: &Path, headers: &[String], row_ids: &[RowId], rows: &[Vec], + structure: Option<&[RowStructure]>, history: &[crate::history::Entry], ) -> Result<()> { if headers.is_empty() { return Ok(()); } - let conn = open_rw(path)?; - conn.execute_batch("BEGIN; DROP TABLE IF EXISTS dataset_main;") + let mut conn = open_rw(path)?; + let had_structure = table_exists(&conn, "__row_structure")?; + let structure = match structure { + Some(structure) => structure.to_vec(), + None if had_structure => read_row_structure_from(&conn)?, + None => Vec::new(), + }; + let structure = retained_row_structure(structure, row_ids); + // A flat v3 project stays v3 until it has an arrangement worth storing. + let write_structure = had_structure || !is_flat(&structure, row_ids); + if write_structure { + validate_row_structure(&structure)?; + } + let tx = conn.transaction().context("Begin dataset rewrite")?; + tx.execute_batch("DROP TABLE IF EXISTS __row_structure; DROP TABLE IF EXISTS dataset_main;") .context("Begin dataset rewrite")?; - create_and_fill_dataset(&conn, headers, Some(row_ids), rows)?; - crate::history::append(&conn, history)?; - conn.execute_batch("COMMIT") - .context("Commit dataset rewrite")?; - Ok(()) + create_and_fill_dataset(&tx, headers, Some(row_ids), rows)?; + if write_structure { + tx.execute_batch(ROW_STRUCTURE_DDL) + .context("Recreate __row_structure")?; + insert_row_structure(&tx, &structure)?; + tx.pragma_update(None, "user_version", QRATE_SCHEMA_VERSION) + .context("Set user_version")?; + } + crate::history::append(&tx, history)?; + tx.commit().context("Commit dataset rewrite") +} + +fn is_flat(structure: &[RowStructure], row_ids: &[RowId]) -> bool { + let Some(first) = structure.first() else { + return true; + }; + let position: HashMap<_, _> = row_ids.iter().enumerate().map(|(i, id)| (*id, i)).collect(); + let mut by_order: Vec<_> = structure.iter().collect(); + by_order.sort_by_key(|row| (row.sibling_order, row.row_id)); + structure.iter().all(|row| { + row.parent_id.is_none() && row.source_path.is_none() && row.level_key == first.level_key + }) && by_order + .windows(2) + .all(|pair| position.get(&pair[0].row_id) < position.get(&pair[1].row_id)) +} + +fn retained_row_structure(structure: Vec, row_ids: &[RowId]) -> Vec { + let retained: HashSet<_> = row_ids.iter().copied().collect(); + let mut structure: Vec<_> = structure + .into_iter() + .filter(|row| retained.contains(&row.row_id)) + .map(|mut row| { + if row.parent_id.is_some_and(|id| !retained.contains(&id)) { + row.parent_id = None; + } + row + }) + .collect(); + structure.sort_by_key(|row| (row.parent_id, row.sibling_order, row.row_id)); + let mut next_order = HashMap::, i64>::new(); + for row in &mut structure { + let order = next_order.entry(row.parent_id).or_default(); + row.sibling_order = *order; + *order += 1; + } + structure } /// Quotes each header as a SQL identifier, de-duplicating case-insensitively @@ -1302,7 +1528,7 @@ mod tests { vec!["1".to_string(), "Edited".to_string()], vec!["2".to_string(), "Second".to_string()], ]; - save_dataset(&path, &headers, &[1, 2], &edited, &[]).unwrap(); + save_dataset(&path, &headers, &[1, 2], &edited, None, &[]).unwrap(); let data = load_project_file(&path).unwrap(); assert_eq!(data.headers, headers); @@ -1313,6 +1539,189 @@ mod tests { assert!(!path.with_extension("qrate-wal").exists()); } + #[test] + fn row_structure_round_trips_and_rejects_cycles() { + let path = tempfile("structure.qrate"); + create_project_file( + &path, + &ProjectSpec { + name: "Structure", + source: "Folder", + headers: &["Title".into()], + rows: &[vec!["Series".into()], vec!["Item".into()]], + ..Default::default() + }, + ) + .unwrap(); + let structure = vec![ + RowStructure { + row_id: 1, + parent_id: None, + level_key: "series".into(), + sibling_order: 0, + source_path: Some("photographs".into()), + source_kind: Some(SourceKind::Directory), + }, + RowStructure { + row_id: 2, + parent_id: Some(1), + level_key: "item".into(), + sibling_order: 0, + source_path: Some("photographs/001.jpg".into()), + source_kind: Some(SourceKind::File), + }, + ]; + write_row_structure(&path, &structure).unwrap(); + assert_eq!(read_row_structure(&path).unwrap(), structure); + + let cycle = vec![ + RowStructure { + parent_id: Some(2), + ..structure[0].clone() + }, + structure[1].clone(), + ]; + assert!(write_row_structure(&path, &cycle).is_err()); + assert_eq!(read_row_structure(&path).unwrap(), structure); + } + + #[test] + fn opening_v3_does_not_create_structure_until_its_first_write() { + let path = tempfile("lazy-v4.qrate"); + create_project_file( + &path, + &ProjectSpec { + name: "Legacy", + source: "CSV", + headers: &["Title".into()], + rows: &[vec!["Item".into()]], + ..Default::default() + }, + ) + .unwrap(); + { + let conn = open_rw(&path).unwrap(); + conn.execute_batch("DROP TABLE __row_structure; PRAGMA user_version = 3") + .unwrap(); + } + + let root = |row_id, sibling_order| RowStructure { + row_id, + parent_id: None, + level_key: "item".into(), + sibling_order, + source_path: None, + source_kind: None, + }; + let headers = ["Title".to_string()]; + assert!(read_row_structure(&path).unwrap().is_empty()); + save_dataset( + &path, + &headers, + &[1], + &[vec!["Edited".into()]], + Some(&[root(1, 0)]), + &[], + ) + .unwrap(); + let conn = open_ro(&path).unwrap(); + assert!(!table_exists(&conn, "__row_structure").unwrap()); + drop(conn); + + // The child is new in this save: its structure has to land with its row, not before it. + let nested = [ + root(1, 0), + RowStructure { + parent_id: Some(1), + ..root(2, 0) + }, + ]; + save_dataset( + &path, + &headers, + &[1, 2], + &[vec!["Series".into()], vec!["Child".into()]], + Some(&nested), + &[], + ) + .unwrap(); + assert_eq!(read_row_structure(&path).unwrap(), nested); + let conn = open_ro(&path).unwrap(); + assert!(table_exists(&conn, "__row_structure").unwrap()); + assert_eq!( + conn.query_row("PRAGMA user_version", [], |row| row.get::<_, i32>(0)) + .unwrap(), + 4 + ); + } + + #[test] + fn dataset_save_preserves_retained_structure_and_promotes_orphans() { + let path = tempfile("save-structure.qrate"); + let headers = vec!["Title".into()]; + create_project_file( + &path, + &ProjectSpec { + name: "Save structure", + source: "Folder", + headers: &headers, + rows: &[vec!["Series".into()], vec!["A".into()], vec!["B".into()]], + ..Default::default() + }, + ) + .unwrap(); + write_row_structure( + &path, + &[ + RowStructure { + row_id: 1, + parent_id: None, + level_key: "series".into(), + sibling_order: 0, + source_path: None, + source_kind: None, + }, + RowStructure { + row_id: 2, + parent_id: Some(1), + level_key: "item".into(), + sibling_order: 0, + source_path: None, + source_kind: None, + }, + RowStructure { + row_id: 3, + parent_id: Some(1), + level_key: "item".into(), + sibling_order: 1, + source_path: None, + source_kind: None, + }, + ], + ) + .unwrap(); + + save_dataset( + &path, + &headers, + &[2, 3], + &[vec!["A".into()], vec!["B".into()]], + None, + &[], + ) + .unwrap(); + let structure = read_row_structure(&path).unwrap(); + assert_eq!(structure.len(), 2); + assert!(structure.iter().all(|row| row.parent_id.is_none())); + assert_eq!( + structure + .iter() + .map(|row| row.sibling_order) + .collect::>(), + [0, 1] + ); + } + #[test] fn notes_follow_stable_rows_through_a_top_insert_and_reload() { let path = tempfile("stable-row-notes.qrate"); @@ -1360,7 +1769,15 @@ mod tests { let mut inserted = vec![vec!["new item".into()]]; inserted.extend(rows); - save_dataset(&path, &headers, &[7, 1, 2, 3, 4, 5, 6], &inserted, &[]).unwrap(); + save_dataset( + &path, + &headers, + &[7, 1, 2, 3, 4, 5, 6], + &inserted, + None, + &[], + ) + .unwrap(); let reopened = load_project_file(&path).unwrap(); assert_eq!(reopened.row_ids, vec![7, 1, 2, 3, 4, 5, 6]); diff --git a/crates/table/Cargo.toml b/crates/table/Cargo.toml index 8360e352..1092d069 100644 --- a/crates/table/Cargo.toml +++ b/crates/table/Cargo.toml @@ -13,6 +13,7 @@ serde.workspace = true serde_json.workspace = true ai = { path = "../ai" } diagnostics = { path = "../diagnostics" } +file-ingest = { path = "../file-ingest" } plugin-api = { path = "../plugin-api" } settings = { path = "../settings" } plugin-host = { path = "../plugin-host" } diff --git a/crates/table/src/cell.rs b/crates/table/src/cell.rs index 34cc46e9..47b98f01 100644 --- a/crates/table/src/cell.rs +++ b/crates/table/src/cell.rs @@ -6,9 +6,9 @@ use gpui::prelude::FluentBuilder as _; use gpui::{ - AnyElement, BorderStyle, Bounds, Context, ElementId, InteractiveElement as _, IntoElement, - MouseButton, ParentElement as _, Pixels, StatefulInteractiveElement as _, Styled as _, Window, - canvas, div, fill, outline, point, px, size, + AnyElement, BorderStyle, Bounds, Context, ElementId, ExternalPaths, InteractiveElement as _, + IntoElement, MouseButton, ParentElement as _, Pixels, StatefulInteractiveElement as _, + Styled as _, Window, canvas, div, fill, outline, point, px, size, }; use gpui_component::menu::ContextMenuExt as _; use gpui_component::{ActiveTheme as _, Icon, IconName, Sizable as _, h_flex, table::TableState}; @@ -90,6 +90,29 @@ pub(crate) fn render_cell( // Own the containing block for the capture canvas below: without this it resolves against // the library's cell div, whose padding makes "the bounds" ambiguous. .relative() + .when(is_filename, |cell| { + cell.can_drop(|value, _, _| { + value + .downcast_ref::() + .is_some_and(|paths| paths.paths().len() == 1 && paths.paths()[0].is_file()) + }) + .drag_over::(|style, _, _, cx| style.bg(cx.theme().secondary_hover)) + .on_drop(move |paths: &ExternalPaths, window, cx| { + let Some(path) = paths.paths().first() else { + return; + }; + let text = file_ingest::normalized_path(path); + window.defer(cx, move |_, cx| { + crate::write_cell( + row_ix, + col_ix, + text.into(), + settings::history::Origin::Typed, + cx, + ) + }); + }) + }) .when(is_filename, |cell| { cell.child( h_flex() diff --git a/crates/table/src/delegate.rs b/crates/table/src/delegate.rs index 924a5d90..7607b51a 100644 --- a/crates/table/src/delegate.rs +++ b/crates/table/src/delegate.rs @@ -16,7 +16,13 @@ use diagnostics::{DATASET_MAIN, Location}; use settings::history::{Change, Entry, EntryId, Origin}; use crate::history::{Cells, Col, History, Row, Step}; -use crate::{cell, editing::EditState, filter, row_index}; +use crate::{ + cell, + editing::EditState, + filter, + hierarchy::{Hierarchy, Placement}, + row_index, +}; /// Emitted whenever the table's selection, a cell's text, or the column layout changes, so /// cross-crate listeners know to re-render. @@ -85,6 +91,10 @@ pub struct QrateTableDelegate { filtered_rows: Vec, /// A visual or linked-file search's hits as source rows, in the order the view shows them. search_rows: Option>, + /// Hierarchy depth parallel to `visible_rows`. + visible_depths: Vec, + hierarchy: Hierarchy, + default_level: String, /// Per-data-column set of *excluded* cell values, parallel to `columns`. Empty = no filter. filters: Vec>, /// Whether each column offers a filter dropdown at all, parallel to `columns`. Off for every @@ -128,6 +138,9 @@ impl QrateTableDelegate { visible_rows: Vec::new(), filtered_rows: Vec::new(), search_rows: None, + visible_depths: Vec::new(), + hierarchy: Hierarchy::default(), + default_level: "item".into(), filters: Vec::new(), filters_enabled: Vec::new(), subdelimiter: SharedString::default(), @@ -174,6 +187,56 @@ impl QrateTableDelegate { &self.visible_rows } + pub fn row_depth(&self, view: usize) -> usize { + self.visible_depths.get(view).copied().unwrap_or_default() + } + + pub(crate) fn row_child_count(&self, source: usize) -> usize { + self.row_ids + .get(source) + .map_or(0, |row_id| self.hierarchy.child_count(*row_id)) + } + + pub(crate) fn row_has_children(&self, source: usize) -> bool { + self.row_ids + .get(source) + .is_some_and(|row_id| self.hierarchy.has_children(*row_id)) + } + + pub(crate) fn row_expanded(&self, source: usize) -> bool { + self.row_ids + .get(source) + .is_some_and(|row_id| self.hierarchy.is_expanded(*row_id)) + } + + pub(crate) fn toggle_expanded(&mut self, source: usize) { + let Some(row_id) = self.row_ids.get(source).copied() else { + return; + }; + self.hierarchy + .set_expanded(row_id, !self.hierarchy.is_expanded(row_id)); + self.recompute_visible(); + } + + pub(crate) fn expand_all(&mut self) { + self.hierarchy.expand_all(); + self.recompute_visible(); + } + + pub(crate) fn collapse_all(&mut self) { + self.hierarchy.collapse_all(); + self.recompute_visible(); + } + + pub(crate) fn expanded_rows(&self) -> Vec { + self.hierarchy.expanded_rows() + } + + pub(crate) fn restore_expanded(&mut self, row_ids: &[settings::project::RowId]) { + self.hierarchy.restore_expanded(row_ids); + self.recompute_visible(); + } + /// Source→view, the inverse of [`source`](Self::source). `None` when a filter currently hides /// the row — a diagnostic can point at a row the user has narrowed away. pub fn view_row(&self, source: usize) -> Option { @@ -214,16 +277,53 @@ impl QrateTableDelegate { fn recompute_visible(&mut self) { self.filtered_rows = compute_visible_rows(&self.rows, &self.filters, &self.subdelimiter); - self.visible_rows = match &self.search_rows { - Some(hits) => { - let kept: HashSet = self.filtered_rows.iter().copied().collect(); - hits.iter() - .copied() - .filter(|row| kept.contains(row)) - .collect() + if let Some(hits) = &self.search_rows { + let kept: HashSet = self.filtered_rows.iter().copied().collect(); + let all_ids = self.row_ids.iter().copied().collect(); + let depths: HashMap<_, _> = self + .hierarchy + .projection(Some(&all_ids)) + .into_iter() + .collect(); + self.visible_rows = hits + .iter() + .copied() + .filter(|row| kept.contains(row)) + .collect(); + self.visible_depths = self + .visible_rows + .iter() + .map(|source| { + self.row_ids + .get(*source) + .and_then(|row_id| depths.get(row_id)) + .copied() + .unwrap_or_default() + }) + .collect(); + } else { + let matching_ids = (self.filtered_rows.len() != self.rows.len()).then(|| { + self.filtered_rows + .iter() + .filter_map(|source| self.row_ids.get(*source).copied()) + .collect::>() + }); + let source_by_id: HashMap<_, _> = self + .row_ids + .iter() + .enumerate() + .map(|(source, row_id)| (*row_id, source)) + .collect(); + let projection = self.hierarchy.projection(matching_ids.as_ref()); + self.visible_rows.clear(); + self.visible_depths.clear(); + for (row_id, depth) in projection { + if let Some(source) = source_by_id.get(&row_id) { + self.visible_rows.push(*source); + self.visible_depths.push(depth); + } } - None => self.filtered_rows.clone(), - }; + } // `range` is in view coordinates, which this just redefined. self.range = None; } @@ -508,12 +608,28 @@ impl QrateTableDelegate { self.filters_enabled = vec![false; self.columns.len()]; self.search_rows = None; self.filtered_rows = (0..self.rows.len()).collect(); - self.visible_rows = self.filtered_rows.clone(); + self.visible_rows = (0..self.rows.len()).collect(); + self.visible_depths = vec![0; self.rows.len()]; + self.hierarchy = Hierarchy::from_rows(&self.row_ids, &[], &self.default_level); // Stale — indexes into the old row set; `TablePanel` re-resolves right after. self.image_paths = vec![None; self.rows.len()]; self.values_generation += 1; } + pub fn set_structure( + &mut self, + structure: &[settings::project::RowStructure], + default_level: &str, + ) { + self.default_level = default_level.into(); + self.hierarchy = Hierarchy::from_rows(&self.row_ids, structure, default_level); + self.recompute_visible(); + } + + pub fn row_structure(&self) -> &[settings::project::RowStructure] { + self.hierarchy.rows() + } + /// Replaces the per-row resolved image paths. A length mismatch (a stale call racing a newer /// `set_data`) is ignored rather than panicking. pub fn set_image_paths(&mut self, paths: Vec>) { @@ -697,16 +813,27 @@ impl QrateTableDelegate { after: after.to_string(), }) .collect(), - Step::RowsAdded { at, rows } => rows - .iter() - .enumerate() - .map(|(offset, row)| Change::RowAdded { - row: row.id, - position: at + offset, - cells: named(&row.cells), - }) - .collect(), - Step::RowsRemoved(rows) => rows + Step::RowsAdded { + at, rows, cells, .. + } => { + let mut changes: Vec<_> = rows + .iter() + .enumerate() + .map(|(offset, row)| Change::RowAdded { + row: row.id, + position: at + offset, + cells: named(&row.cells), + }) + .collect(); + changes.extend(cells.iter().map(|(row, col, before, after)| Change::Cell { + row: row_id(*row), + column: self.column_name(*col).to_string(), + before: before.to_string(), + after: after.to_string(), + })); + changes + } + Step::RowsRemoved { rows, .. } => rows .iter() .map(|(at, row)| Change::RowRemoved { row: row.id, @@ -734,7 +861,7 @@ impl QrateTableDelegate { to: *to, }], // A batch's steps are read one by one, each in its own state, as they are replayed. - Step::Batch(_) => Vec::new(), + Step::Batch(_) | Step::Hierarchy { .. } => Vec::new(), } } @@ -813,12 +940,27 @@ impl QrateTableDelegate { image: None, }]; let at = (*position).min(self.rows.len()); + let before_structure = self.hierarchy.rows().to_vec(); self.splice_rows(at, &rows); - Some(Step::RowsAdded { at, rows }) + let after_structure = self.hierarchy.rows().to_vec(); + Some(Step::RowsAdded { + at, + rows, + cells: Vec::new(), + before_structure, + after_structure, + }) } Change::RowRemoved { row, .. } => { let at = find(&self.row_ids, *row)?; - Some(Step::RowsRemoved(self.cut_rows(&[at]))) + let before_structure = self.hierarchy.rows().to_vec(); + let rows = self.cut_rows(&[at]); + let after_structure = self.hierarchy.rows().to_vec(); + Some(Step::RowsRemoved { + rows, + before_structure, + after_structure, + }) } Change::ColumnAdded { column, @@ -899,14 +1041,38 @@ impl QrateTableDelegate { self.set_cell(*row, *col, text.clone()); } } - Step::RowsAdded { at, rows } => { + Step::RowsAdded { + at, + rows, + cells, + before_structure, + after_structure, + } => { if forward { self.splice_rows(*at, rows); } else { self.cut_rows(&(*at..at + rows.len()).collect::>()); } + for (row, col, before, after) in cells { + let text = if forward { after } else { before }; + self.set_cell(*row, *col, text.clone()); + } + self.hierarchy.replace_rows( + &self.row_ids, + if forward { + after_structure + } else { + before_structure + }, + &self.default_level, + ); + self.recompute_visible(); } - Step::RowsRemoved(rows) => { + Step::RowsRemoved { + rows, + before_structure, + after_structure, + } => { if forward { self.cut_rows(&rows.iter().map(|(at, _)| *at).collect::>()); } else { @@ -914,6 +1080,16 @@ impl QrateTableDelegate { self.splice_rows(*at, std::slice::from_ref(row)); } } + self.hierarchy.replace_rows( + &self.row_ids, + if forward { + after_structure + } else { + before_structure + }, + &self.default_level, + ); + self.recompute_visible(); } Step::ColumnAdded { at, col } => { if forward { @@ -943,6 +1119,14 @@ impl QrateTableDelegate { .rev() .for_each(|s| self.replay(s, false, changes)), }, + Step::Hierarchy { before, after } => { + self.hierarchy.replace_rows( + &self.row_ids, + if forward { after } else { before }, + &self.default_level, + ); + self.recompute_visible(); + } } if forward { changes.extend(self.changes(step)); @@ -1040,6 +1224,7 @@ impl QrateTableDelegate { fn rows_changed(&mut self) { // Hits are source rows, which an insert or delete just shifted. self.search_rows = None; + self.hierarchy.reconcile(&self.row_ids, &self.default_level); self.recompute_visible(); self.editing = EditState::Idle; self.values_generation += 1; @@ -1077,6 +1262,7 @@ impl QrateTableDelegate { /// Insert a blank row at `at`, or a copy of `source` when duplicating a row. pub(crate) fn insert_rows(&mut self, at: usize, source: Option) { + let before_structure = self.hierarchy.rows().to_vec(); let cells = match source.and_then(|r| self.rows.get(r)) { Some(row) => row.clone(), None => vec![SharedString::default(); self.columns.len()], @@ -1085,13 +1271,243 @@ impl QrateTableDelegate { let id = self.fresh_row_id(); let rows = vec![Row { id, cells, image }]; self.splice_rows(at, &rows); - self.record(Step::RowsAdded { at, rows }, Origin::Structure); + // A duplicate follows its original; a blank row takes the place of the row it pushed down. + let neighbour = source + .map(|row| (row, false)) + .or_else(|| (at + 1 < self.rows.len()).then_some((at + 1, true))) + .or_else(|| at.checked_sub(1).map(|row| (row, false))); + if let Some((neighbour, before)) = neighbour + && let Some(neighbour_id) = self.row_id(neighbour) + { + let level = self.hierarchy.level(neighbour_id).map(str::to_owned); + let placement = match before { + true => Placement::Before(neighbour_id), + false => Placement::After(neighbour_id), + }; + match self.hierarchy.move_row(id, placement) { + Ok(()) => { + if let Some(level) = level { + let _ = self.hierarchy.set_level(id, &level); + } + } + Err(error) => log::warn!( + "New row {id} was added at the end of the table instead of beside row {}: {error:?}", + neighbour + 1 + ), + } + self.recompute_visible(); + } + let after_structure = self.hierarchy.rows().to_vec(); + self.record( + Step::RowsAdded { + at, + rows, + cells: Vec::new(), + before_structure, + after_structure, + }, + Origin::Structure, + ); + } + + /// What the project already holds, for [`file_ingest::duplicates::resolve`]: every row's linked + /// file, by stored source path and by the keys its Filename cell can be looked up under. + pub(crate) fn existing_components( + &self, + file_col: Option, + files_root: Option<&Path>, + ) -> Vec { + let sources: HashMap<_, _> = self + .hierarchy + .rows() + .iter() + .filter_map(|row| Some((row.row_id, row.source_path.clone()?))) + .collect(); + self.row_ids + .iter() + .enumerate() + .map(|(source, row_id)| { + let stored = sources.get(row_id).map(PathBuf::from).map(|path| { + match (path.is_absolute(), files_root) { + (false, Some(root)) => root.join(path), + _ => path, + } + }); + let cell = file_col + .and_then(|col| self.rows.get(source)?.get(col)) + .map(|cell| cell.to_string()); + file_ingest::duplicates::ExistingComponent { + key: *row_id as u64, + absolute_source: stored.or_else(|| { + cell.as_deref() + .map(PathBuf::from) + .filter(|path| path.is_absolute()) + }), + filename_keys: cell + .map(|cell| settings::filenames::lookup_keys(&cell)) + .unwrap_or_default(), + } + }) + .collect() + } + + /// Applies a resolved import as one undo step: new components become rows, and a component the + /// policy resolved onto an existing row re-links that row rather than copying it. + pub(crate) fn append_components( + &mut self, + resolved: &file_ingest::duplicates::ResolvedPlan, + title_col: Option, + file_col: Option, + destination_parent: Option, + files_root: Option<&Path>, + ) -> usize { + use file_ingest::duplicates::{Parent, Resolution}; + + if resolved.plan.components.is_empty() { + return 0; + } + let at = self.rows.len(); + let before_structure = self.hierarchy.rows().to_vec(); + let destination_parent = destination_parent.and_then(|source| self.row_id(source)); + let mut next_order = before_structure + .iter() + .filter(|row| row.parent_id == destination_parent) + .count() as i64; + let source_path = |component: &file_ingest::PlannedComponent| { + file_ingest::normalized_path( + files_root + .and_then(|root| component.absolute_path.strip_prefix(root).ok()) + .filter(|relative| !relative.as_os_str().is_empty()) + .unwrap_or(&component.absolute_path), + ) + }; + + // Row ids first: a component's parent may be resolved onto a row the project already has. + let mut ids = Vec::with_capacity(resolved.plan.components.len()); + for resolution in &resolved.resolutions { + let id = match resolution { + Resolution::Update { existing } | Resolution::Skip { existing } => *existing as i64, + _ => self.fresh_row_id(), + }; + ids.push(id); + } + + let mut rows = Vec::new(); + let mut cells = Vec::new(); + let mut after_structure = before_structure.clone(); + for (index, component) in resolved.plan.components.iter().enumerate() { + let is_file = component.kind == file_ingest::EntryKind::File; + let parent_id = match resolved.parents[index] { + Some(Parent::Planned(parent)) => ids.get(parent).copied(), + Some(Parent::Existing(existing)) => Some(existing as i64), + None => destination_parent, + }; + match &resolved.resolutions[index] { + Resolution::Skip { .. } => continue, + Resolution::Update { existing } => { + let Some(source) = self.row_ids.iter().position(|id| *id as u64 == *existing) + else { + continue; + }; + // Only the link moves. The metadata on this row is the archivist's. + if is_file + && let Some(col) = file_col + && let Some(before) = self.rows.get(source).and_then(|row| row.get(col)) + { + let after = SharedString::from(file_ingest::normalized_path( + &component.absolute_path, + )); + if *before != after { + cells.push((source, col, before.clone(), after)); + } + } + if let Some(row) = after_structure + .iter_mut() + .find(|row| row.row_id as u64 == *existing) + { + row.source_path = Some(source_path(component)); + } + } + Resolution::Create | Resolution::Ambiguous { .. } => { + let mut new_cells = vec![SharedString::default(); self.columns.len()]; + if let Some(col) = title_col.filter(|col| *col < new_cells.len()) { + new_cells[col] = component.title.clone().into(); + } + if is_file && let Some(col) = file_col.filter(|col| *col < new_cells.len()) { + new_cells[col] = + file_ingest::normalized_path(&component.absolute_path).into(); + } + rows.push(Row { + id: ids[index], + cells: new_cells, + image: is_file.then(|| component.absolute_path.clone()), + }); + let order = match component.parent { + Some(_) => index as i64, + None => { + next_order += 1; + next_order - 1 + } + }; + after_structure.push(settings::project::RowStructure { + row_id: ids[index], + parent_id, + level_key: component.level_key.clone(), + sibling_order: order, + source_path: Some(source_path(component)), + source_kind: Some(match component.kind { + file_ingest::EntryKind::File => settings::project::SourceKind::File, + file_ingest::EntryKind::Directory => { + settings::project::SourceKind::Directory + } + }), + }); + } + } + } + + let added = rows.len(); + self.splice_rows(at, &rows); + for (row, col, _, after) in &cells { + self.set_cell(*row, *col, after.clone()); + } + self.hierarchy + .replace_rows(&self.row_ids, &after_structure, &self.default_level); + self.recompute_visible(); + self.record( + Step::RowsAdded { + at, + rows, + cells, + before_structure, + after_structure, + }, + Origin::Structure, + ); + added } /// Delete the rows at `ats` as one undo step, carrying their cells and photos on it. pub(crate) fn remove_rows(&mut self, ats: &[usize]) { + let before_structure = self.hierarchy.rows().to_vec(); + let removed_ids: Vec<_> = ats.iter().filter_map(|at| self.row_id(*at)).collect(); + for row_id in removed_ids { + if let Err(error) = self.hierarchy.delete(row_id) { + log::warn!( + "Deleted row {row_id} had no archival structure, so its children kept their place: {error:?}" + ); + } + } let removed = self.cut_rows(ats); - self.record(Step::RowsRemoved(removed), Origin::Structure); + let after_structure = self.hierarchy.rows().to_vec(); + self.record( + Step::RowsRemoved { + rows: removed, + before_structure, + after_structure, + }, + Origin::Structure, + ); } pub(crate) fn row_id(&self, source: usize) -> Option { @@ -1102,6 +1518,133 @@ impl QrateTableDelegate { &self.row_ids } + pub(crate) fn indent_row(&mut self, source: usize) -> Result<(), crate::hierarchy::Error> { + let row_id = self + .row_ids + .get(source) + .copied() + .ok_or(crate::hierarchy::Error::UnknownRow(source as i64))?; + self.edit_hierarchy(|hierarchy| hierarchy.indent(row_id)) + } + + pub(crate) fn outdent_row(&mut self, source: usize) -> Result<(), crate::hierarchy::Error> { + let row_id = self + .row_ids + .get(source) + .copied() + .ok_or(crate::hierarchy::Error::UnknownRow(source as i64))?; + self.edit_hierarchy(|hierarchy| hierarchy.outdent(row_id)) + } + + pub(crate) fn reparent_row( + &mut self, + source: usize, + parent_source: Option, + ) -> Result<(), crate::hierarchy::Error> { + let row_id = self + .row_ids + .get(source) + .copied() + .ok_or(crate::hierarchy::Error::UnknownRow(source as i64))?; + let placement = match parent_source { + Some(parent) => Placement::ChildOf( + self.row_ids + .get(parent) + .copied() + .ok_or(crate::hierarchy::Error::UnknownRow(parent as i64))?, + ), + None => Placement::Root, + }; + self.edit_hierarchy(|hierarchy| hierarchy.move_row(row_id, placement)) + } + + pub(crate) fn move_row_relative( + &mut self, + source: usize, + target: usize, + placement: crate::RowPlacement, + ) -> Result<(), crate::hierarchy::Error> { + let row_id = self + .row_id(source) + .ok_or(crate::hierarchy::Error::UnknownRow(source as i64))?; + let target_id = self + .row_id(target) + .ok_or(crate::hierarchy::Error::UnknownRow(target as i64))?; + let placement = match placement { + crate::RowPlacement::Before => Placement::Before(target_id), + crate::RowPlacement::Child => Placement::ChildOf(target_id), + crate::RowPlacement::After => Placement::After(target_id), + }; + self.edit_hierarchy(|hierarchy| hierarchy.move_row(row_id, placement)) + } + + /// Wrap `rows` in a new blank parent that takes the first one's place, as one undo step. A row + /// whose ancestor is also picked travels with that ancestor rather than being pulled out of it. + pub(crate) fn group_rows(&mut self, rows: &[usize]) -> Result<(), crate::hierarchy::Error> { + let Some(&first) = rows.first() else { + return Ok(()); + }; + let picked = rows + .iter() + .map(|&row| { + self.row_id(row) + .ok_or(crate::hierarchy::Error::UnknownRow(row as i64)) + }) + .collect::, _>>()?; + let nested: HashSet<_> = picked + .iter() + .filter_map(|&id| self.hierarchy.subtree(id).ok()) + .flat_map(|subtree| subtree.into_iter().skip(1)) + .collect(); + self.insert_rows(first, None); + let group = self.row_ids[first]; + for id in picked.into_iter().filter(|id| !nested.contains(id)) { + // A fresh, childless parent cannot be a descendant of anything picked. + if let Err(error) = self.hierarchy.move_row(id, Placement::ChildOf(group)) { + log::warn!( + "Row {id} stayed where it was instead of joining the new group: {error:?}" + ); + } + } + self.hierarchy.set_expanded(group, true); + let structure = self.hierarchy.rows().to_vec(); + if let Some(Step::RowsAdded { + after_structure, .. + }) = self.history.last_mut() + { + *after_structure = structure; + } + self.recompute_visible(); + Ok(()) + } + + pub(crate) fn subtree_sources(&self, source: usize) -> Vec { + let Some(row_id) = self.row_ids.get(source).copied() else { + return Vec::new(); + }; + let Ok(subtree) = self.hierarchy.subtree(row_id) else { + return Vec::new(); + }; + let subtree: HashSet<_> = subtree.into_iter().collect(); + self.row_ids + .iter() + .enumerate() + .filter_map(|(source, row_id)| subtree.contains(row_id).then_some(source)) + .collect() + } + + fn edit_hierarchy( + &mut self, + edit: impl FnOnce(&mut Hierarchy) -> Result<(), crate::hierarchy::Error>, + ) -> Result<(), crate::hierarchy::Error> { + let before = self.hierarchy.rows().to_vec(); + edit(&mut self.hierarchy)?; + let after = self.hierarchy.rows().to_vec(); + self.record(Step::Hierarchy { before, after }, Origin::Structure); + self.recompute_visible(); + Ok(()) + } + fn fresh_row_id(&mut self) -> settings::project::RowId { while self.row_ids.contains(&self.next_row_id) { self.next_row_id = self.next_row_id.checked_add(1).unwrap_or(1); @@ -1645,7 +2188,7 @@ impl TableDelegate for QrateTableDelegate { // it while its row holds the active (edited/selected) cell. let highlighted = self.active_cell().is_some_and(|(r, _)| r == source) || self.selected_rows.contains(&source); - row_index::render_td(self, source, highlighted, cx) + row_index::render_td(self, row_ix, source, highlighted, cx) } else { // A ⌘-clicked row paints across every column, which is what makes a discontiguous // selection read as a set of *items* rather than a column of tinted cells. @@ -2029,6 +2572,288 @@ mod app_tests { }); } + #[gpui::test] + fn hierarchy_projection_maps_view_rows_to_sources_and_depths(cx: &mut TestAppContext) { + let state = table(cx); + cx.update(|cx| { + state.update(cx, |state, _| { + let delegate = state.delegate_mut(); + delegate.set_structure( + &[ + settings::project::RowStructure { + row_id: 1, + parent_id: None, + level_key: "series".into(), + sibling_order: 0, + source_path: None, + source_kind: None, + }, + settings::project::RowStructure { + row_id: 2, + parent_id: Some(1), + level_key: "item".into(), + sibling_order: 0, + source_path: None, + source_kind: None, + }, + ], + "item", + ); + assert_eq!(delegate.visible(), &[0, 2, 3]); + delegate.toggle_expanded(0); + assert_eq!(delegate.visible(), &[0, 1, 2, 3]); + assert_eq!(delegate.row_depth(1), 1); + assert_eq!(delegate.source(1), Some(1)); + }); + }); + } + + #[gpui::test] + fn dropped_components_append_as_one_undoable_hierarchy(cx: &mut TestAppContext) { + let state = table(cx); + cx.update(|cx| { + state.update(cx, |state, _| { + let plan = file_ingest::ImportPlan { + components: vec![ + file_ingest::PlannedComponent { + title: "Series".into(), + absolute_path: "C:/archive/Series".into(), + source_path: "Series".into(), + kind: file_ingest::EntryKind::Directory, + parent: None, + level_key: "series".into(), + }, + file_ingest::PlannedComponent { + title: "image.jpg".into(), + absolute_path: "C:/archive/Series/image.jpg".into(), + source_path: "Series/image.jpg".into(), + kind: file_ingest::EntryKind::File, + parent: Some(0), + level_key: "item".into(), + }, + ], + warnings: Vec::new(), + }; + let root = std::path::Path::new("C:/archive"); + let delegate = state.delegate_mut(); + let resolved = file_ingest::duplicates::resolve( + plan, + &delegate.existing_components(Some(0), Some(root)), + settings::filenames::keys, + file_ingest::duplicates::DuplicatePolicy::Skip, + ); + assert_eq!( + delegate.append_components(&resolved, Some(1), Some(0), None, Some(root)), + 2 + ); + assert_eq!(delegate.row_count(), 6); + // The dropped series queues after the existing roots; its collapsed child stays hidden. + assert_eq!(delegate.visible(), &[0, 1, 2, 3, 4]); + let structure = delegate.row_structure(); + assert_eq!(structure[5].parent_id, Some(structure[4].row_id)); + assert_eq!( + structure[5].source_path.as_deref(), + Some("Series/image.jpg") + ); + assert_eq!(delegate.cell(5, 1).map(AsRef::as_ref), Some("image.jpg")); + assert!(delegate.undo().is_some()); + assert_eq!(delegate.row_count(), 4); + assert!(delegate.redo().is_some()); + assert_eq!(delegate.row_count(), 6); + }); + }); + } + + /// ASNT-77: a second batch over a folder qrate already holds. + #[gpui::test] + fn re_dropping_a_folder_adds_only_what_is_new(cx: &mut TestAppContext) { + let state = table(cx); + cx.update(|cx| { + state.update(cx, |state, _| { + let root = std::path::Path::new("C:/archive"); + let component = |name: &str, kind, parent| file_ingest::PlannedComponent { + title: name.rsplit('/').next().unwrap_or(name).into(), + absolute_path: format!("C:/archive/{name}").into(), + source_path: name.into(), + kind, + parent, + level_key: "item".into(), + }; + let first = file_ingest::ImportPlan { + components: vec![ + component("Series", file_ingest::EntryKind::Directory, None), + component("Series/one.jpg", file_ingest::EntryKind::File, Some(0)), + ], + warnings: Vec::new(), + }; + let second = file_ingest::ImportPlan { + components: vec![ + component("Series", file_ingest::EntryKind::Directory, None), + component("Series/one.jpg", file_ingest::EntryKind::File, Some(0)), + component("Series/two.jpg", file_ingest::EntryKind::File, Some(0)), + ], + warnings: Vec::new(), + }; + let resolve = |existing: &[file_ingest::duplicates::ExistingComponent], plan| { + file_ingest::duplicates::resolve( + plan, + existing, + settings::filenames::keys, + file_ingest::duplicates::DuplicatePolicy::Skip, + ) + }; + + let delegate = state.delegate_mut(); + let resolved = resolve(&delegate.existing_components(Some(0), Some(root)), first); + delegate.append_components(&resolved, Some(1), Some(0), None, Some(root)); + assert_eq!(delegate.row_count(), 6); + + let resolved = resolve(&delegate.existing_components(Some(0), Some(root)), second); + assert_eq!(resolved.duplicates(), 2); + // Only the new photograph arrives, and it joins the series already in the table. + assert_eq!( + delegate.append_components(&resolved, Some(1), Some(0), None, Some(root)), + 1 + ); + assert_eq!(delegate.row_count(), 7); + let structure = delegate.row_structure(); + assert_eq!(structure[6].parent_id, Some(structure[4].row_id)); + assert!(delegate.undo().is_some()); + assert_eq!(delegate.row_count(), 6); + }); + }); + } + + #[gpui::test] + fn inserted_rows_appear_where_they_were_asked_for(cx: &mut TestAppContext) { + let state = table(cx); + cx.update(|cx| { + state.update(cx, |state, _| { + let delegate = state.delegate_mut(); + delegate.insert_rows(1, None); + assert_eq!(delegate.visible(), &[0, 1, 2, 3, 4]); + delegate.insert_rows(3, Some(2)); + assert_eq!(delegate.visible(), &[0, 1, 2, 3, 4, 5]); + }); + }); + } + + #[gpui::test] + fn deleting_a_middle_parent_promotes_children_into_its_place(cx: &mut TestAppContext) { + let state = table(cx); + cx.update(|cx| { + state.update(cx, |state, _| { + let component = + |row_id, parent_id, sibling_order| settings::project::RowStructure { + row_id, + parent_id, + level_key: "item".into(), + sibling_order, + source_path: None, + source_kind: None, + }; + let delegate = state.delegate_mut(); + // 1 ─┬─ 2 ── 3 + // └─ 4 + delegate.set_structure( + &[ + component(1, None, 0), + component(2, Some(1), 0), + component(3, Some(2), 0), + component(4, Some(1), 1), + ], + "item", + ); + delegate.remove_rows(&[1]); + let parent_of = |id| { + delegate + .row_structure() + .iter() + .find(|row| row.row_id == id) + .map(|row| (row.parent_id, row.sibling_order)) + }; + assert_eq!(parent_of(3), Some((Some(1), 0))); + assert_eq!(parent_of(4), Some((Some(1), 1))); + }); + }); + } + + /// Grouping is one gesture, so the new parent and the moves under it undo together. + #[gpui::test] + fn grouping_rows_adds_one_parent_and_undoes_as_one_step(cx: &mut TestAppContext) { + let state = table(cx); + cx.update(|cx| { + state.update(cx, |state, _| { + let delegate = state.delegate_mut(); + delegate.group_rows(&[1, 2]).unwrap(); + assert_eq!(delegate.row_count(), 5); + let group = delegate.row_ids()[1]; + let parent_of = |delegate: &super::QrateTableDelegate, id| { + delegate + .row_structure() + .iter() + .find(|row| row.row_id == id) + .and_then(|row| row.parent_id) + }; + assert_eq!(parent_of(delegate, 2), Some(group)); + assert_eq!(parent_of(delegate, 3), Some(group)); + assert_eq!(parent_of(delegate, group), None); + assert_eq!(delegate.row_child_count(1), 2); + + delegate.undo(); + assert_eq!(delegate.row_count(), 4); + assert_eq!(parent_of(delegate, 2), None); + assert_eq!(parent_of(delegate, 3), None); + }); + }); + } + + #[gpui::test] + fn hierarchy_edits_and_row_deletion_restore_structure_on_undo(cx: &mut TestAppContext) { + let state = table(cx); + cx.update(|cx| { + state.update(cx, |state, _| { + let delegate = state.delegate_mut(); + delegate.set_structure( + &[ + settings::project::RowStructure { + row_id: 1, + parent_id: None, + level_key: "series".into(), + sibling_order: 0, + source_path: None, + source_kind: None, + }, + settings::project::RowStructure { + row_id: 2, + parent_id: None, + level_key: "item".into(), + sibling_order: 1, + source_path: None, + source_kind: None, + }, + ], + "item", + ); + delegate.indent_row(1).unwrap(); + assert_eq!(delegate.row_structure()[1].parent_id, Some(1)); + delegate.undo(); + assert_eq!(delegate.row_structure()[1].parent_id, None); + + delegate.indent_row(1).unwrap(); + delegate.remove_rows(&[0]); + assert_eq!(delegate.row_structure()[0].parent_id, None); + delegate.undo(); + let child = delegate + .row_structure() + .iter() + .find(|row| row.row_id == 2) + .unwrap(); + assert_eq!(child.parent_id, Some(1)); + }); + }); + } + /// What makes Copy/Cut/Paste/Clear work on a whole row or column: with no rectangle drawn, the /// selection itself is the range. A selected column covers only the rows the filter left /// visible, so copying it gives back what the user can see. diff --git a/crates/table/src/editing.rs b/crates/table/src/editing.rs index 135e31f3..20d37fca 100644 --- a/crates/table/src/editing.rs +++ b/crates/table/src/editing.rs @@ -57,10 +57,21 @@ pub(crate) fn start( cx.spawn_in(window, async move |_, cx| { if let Ok(Ok(Some(paths))) = receiver.await && let Some(path) = paths.first() - && let Some(name) = path.file_name().and_then(|name| name.to_str()) { - let name: SharedString = name.to_owned().into(); let parent = path.parent().map(Path::to_path_buf); + // Relative to the files folder when the file is inside it, the way an imported + // row names its file. A file from anywhere else keeps its absolute path, since a + // bare name there is one the preview resolver can never find. + let root = match had_folder { + true => Some(Path::new(folder.as_ref()).to_path_buf()), + false => parent.clone(), + }; + let name: SharedString = + match root.as_deref().map(|root| path.strip_prefix(root)) { + Some(Ok(relative)) => relative.to_string_lossy().replace('\\', "/"), + _ => path.to_string_lossy().into_owned(), + } + .into(); let _ = cx.update(|_, cx| { // A project with no files folder resolves every row to no file, so the picked // file would name a preview qrate could never find. The first file chosen by @@ -301,6 +312,41 @@ mod tests { }); } + /// A file picked from outside the files folder has no name the folder's index knows, so the + /// cell has to carry where it is or the row previews nothing. + #[gpui::test] + async fn a_file_picked_outside_the_files_folder_still_resolves(cx: &mut TestAppContext) { + let root = std::env::temp_dir().join("qrate-editing-outside-pick"); + let _ = std::fs::remove_dir_all(&root); + let (folder, elsewhere) = (root.join("collection"), root.join("elsewhere")); + std::fs::create_dir_all(&folder).unwrap(); + std::fs::create_dir_all(&elsewhere).unwrap(); + let file = elsewhere.join("loose.jpg"); + std::fs::write(&file, "x").unwrap(); + + let state = panel(cx); + cx.update(|cx| { + settings::project::CurrentProject::set_text( + settings::project::FILES_FOLDER_KEY, + folder.to_string_lossy().into_owned().into(), + cx, + ); + }); + let window = cx.windows()[0]; + cx.update_window(window, |_, window, cx| { + state.update(cx, |state, cx| { + editing::start(state.delegate_mut(), 0, 0, window, cx); + }); + }) + .unwrap(); + cx.simulate_path_prompt_response(|_| Some(vec![file.clone()])); + cx.run_until_parked(); + + cx.update(|cx| { + assert_eq!(state.read(cx).delegate().row_image(0), Some(file.as_path())); + }); + } + /// The other half: without the cancel, that same commit does write — so the assertion above is /// about `cancel` and not about a commit path that never fires. #[gpui::test] diff --git a/crates/table/src/filter.rs b/crates/table/src/filter.rs index 9bb0005e..310c0d20 100644 --- a/crates/table/src/filter.rs +++ b/crates/table/src/filter.rs @@ -101,6 +101,15 @@ pub(crate) fn render_th( .font_semibold() .when(editing_col, |th| th.bg(cx.theme().secondary_hover)) .child(div().flex_1().min_w_0().truncate().child(name)) + // The header is the one string a column is, so double-clicking it edits that string — + // the same gesture that edits a cell. + .on_click(cx.listener(move |state, event: &ClickEvent, window, cx| { + if event.click_count() < 2 { + return; + } + crate::editing::start_rename(state.delegate_mut(), data_col, window, cx); + cx.notify(); + })) .when_some(worst, |th, severity| th.child(note::marker(severity, cx))) .when_some(tip, |th, text| { th.tooltip(move |window, cx| { diff --git a/crates/table/src/hierarchy.rs b/crates/table/src/hierarchy.rs new file mode 100644 index 00000000..5dbd7c8e --- /dev/null +++ b/crates/table/src/hierarchy.rs @@ -0,0 +1,465 @@ +use std::collections::{HashMap, HashSet}; + +use settings::project::{RowId, RowStructure}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Placement { + Before(RowId), + After(RowId), + ChildOf(RowId), + Root, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Error { + UnknownRow(RowId), + CannotMoveIntoDescendant, + NoPreviousSibling, + AlreadyRoot, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct Hierarchy { + rows: Vec, + expanded: HashSet, + /// How many children each parent row has, rebuilt by `normalize` after every structural change. + parents: HashMap, +} + +impl Hierarchy { + pub(crate) fn from_rows( + row_ids: &[RowId], + stored: &[RowStructure], + default_level: &str, + ) -> Self { + let stored: HashMap<_, _> = stored.iter().map(|row| (row.row_id, row)).collect(); + let known: HashSet<_> = row_ids.iter().copied().collect(); + let mut next_root = stored + .values() + .filter(|row| row.parent_id.is_none()) + .map(|row| row.sibling_order + 1) + .max() + .unwrap_or_default(); + let rows = row_ids + .iter() + .map(|row_id| match stored.get(row_id) { + Some(row) => { + let mut row = (*row).clone(); + if row.parent_id.is_some_and(|parent| !known.contains(&parent)) { + row.parent_id = None; + } + row + } + None => { + let row = RowStructure { + row_id: *row_id, + parent_id: None, + level_key: default_level.into(), + sibling_order: next_root, + source_path: None, + source_kind: None, + }; + next_root += 1; + row + } + }) + .collect::>(); + let mut hierarchy = Self { + rows, + ..Self::default() + }; + hierarchy.normalize(); + hierarchy + } + + pub(crate) fn rows(&self) -> &[RowStructure] { + &self.rows + } + + pub(crate) fn reconcile(&mut self, row_ids: &[RowId], default_level: &str) { + let expanded = std::mem::take(&mut self.expanded); + let mut next = Self::from_rows(row_ids, &self.rows, default_level); + next.expanded = expanded + .into_iter() + .filter(|row_id| row_ids.contains(row_id)) + .collect(); + *self = next; + } + + pub(crate) fn replace_rows(&mut self, row_ids: &[RowId], rows: &[RowStructure], default: &str) { + let expanded = std::mem::take(&mut self.expanded); + let mut next = Self::from_rows(row_ids, rows, default); + next.expanded = expanded; + *self = next; + } + + pub(crate) fn subtree(&self, row_id: RowId) -> Result, Error> { + self.row(row_id)?; + let mut rows = vec![row_id]; + rows.extend(self.descendants(row_id)); + Ok(rows) + } + + pub(crate) fn set_expanded(&mut self, row_id: RowId, expanded: bool) { + if expanded { + self.expanded.insert(row_id); + } else { + self.expanded.remove(&row_id); + } + } + + pub(crate) fn is_expanded(&self, row_id: RowId) -> bool { + self.expanded.contains(&row_id) + } + + pub(crate) fn expanded_rows(&self) -> Vec { + let mut rows: Vec<_> = self.expanded.iter().copied().collect(); + rows.sort_unstable(); + rows + } + + pub(crate) fn restore_expanded(&mut self, row_ids: &[RowId]) { + let known: HashSet<_> = self.rows.iter().map(|row| row.row_id).collect(); + self.expanded = row_ids + .iter() + .copied() + .filter(|row_id| known.contains(row_id) && self.has_children(*row_id)) + .collect(); + } + + pub(crate) fn child_count(&self, row_id: RowId) -> usize { + self.parents.get(&row_id).copied().unwrap_or_default() + } + + pub(crate) fn has_children(&self, row_id: RowId) -> bool { + self.parents.contains_key(&row_id) + } + + pub(crate) fn set_level(&mut self, row_id: RowId, level_key: &str) -> Result<(), Error> { + self.row_mut(row_id)?.level_key = level_key.into(); + Ok(()) + } + + pub(crate) fn level(&self, row_id: RowId) -> Option<&str> { + self.row(row_id).ok().map(|row| row.level_key.as_str()) + } + + pub(crate) fn expand_all(&mut self) { + self.expanded = self.parents.keys().copied().collect(); + } + + pub(crate) fn collapse_all(&mut self) { + self.expanded.clear(); + } + + pub(crate) fn projection(&self, matching: Option<&HashSet>) -> Vec<(RowId, usize)> { + let included = matching.map(|matching| { + let by_id: HashMap<_, _> = self.rows.iter().map(|row| (row.row_id, row)).collect(); + let mut included = matching.clone(); + for row_id in matching { + let mut parent = by_id.get(row_id).and_then(|row| row.parent_id); + while let Some(parent_id) = parent { + if !included.insert(parent_id) { + break; + } + parent = by_id.get(&parent_id).and_then(|row| row.parent_id); + } + } + included + }); + let index = self.children_index(); + let mut output = Vec::new(); + self.append_children(&index, None, 0, included.as_ref(), &mut output); + output + } + + fn append_children( + &self, + index: &HashMap, Vec>, + parent: Option, + depth: usize, + included: Option<&HashSet>, + output: &mut Vec<(RowId, usize)>, + ) { + for &child in index.get(&parent).map(Vec::as_slice).unwrap_or_default() { + if included.is_none_or(|included| included.contains(&child)) { + output.push((child, depth)); + } + // `included` holds every ancestor of a match, so a deeper match shows up as an + // included direct child. + let reveal_match = included.is_some_and(|included| { + index + .get(&Some(child)) + .is_some_and(|children| children.iter().any(|row| included.contains(row))) + }); + if self.expanded.contains(&child) || reveal_match { + self.append_children(index, Some(child), depth + 1, included, output); + } + } + } + + pub(crate) fn move_row(&mut self, row_id: RowId, placement: Placement) -> Result<(), Error> { + self.row(row_id)?; + let (parent, at) = match placement { + Placement::Root => (None, self.children(None).len()), + Placement::ChildOf(target) => { + self.ensure_target(row_id, target)?; + (Some(target), self.children(Some(target)).len()) + } + Placement::Before(target) | Placement::After(target) => { + self.ensure_target(row_id, target)?; + let target_row = self.row(target)?; + let mut siblings = self.children(target_row.parent_id); + siblings.retain(|id| *id != row_id); + let target_at = siblings.iter().position(|id| *id == target).unwrap_or(0); + ( + target_row.parent_id, + target_at + usize::from(matches!(placement, Placement::After(_))), + ) + } + }; + let row = self.row_mut(row_id)?; + row.parent_id = parent; + row.sibling_order = at as i64; + self.normalize_with_priority(row_id, at); + Ok(()) + } + + pub(crate) fn indent(&mut self, row_id: RowId) -> Result<(), Error> { + let row = self.row(row_id)?; + let siblings = self.children(row.parent_id); + let at = siblings.iter().position(|id| *id == row_id).unwrap_or(0); + let previous = at + .checked_sub(1) + .and_then(|at| siblings.get(at)) + .copied() + .ok_or(Error::NoPreviousSibling)?; + self.move_row(row_id, Placement::ChildOf(previous)) + } + + pub(crate) fn outdent(&mut self, row_id: RowId) -> Result<(), Error> { + let parent = self.row(row_id)?.parent_id.ok_or(Error::AlreadyRoot)?; + self.move_row(row_id, Placement::After(parent)) + } + + /// Removes one row, leaving its children where it stood. Deleting a whole subtree is the + /// caller removing every row in [`subtree`](Self::subtree), which is one undoable row deletion. + pub(crate) fn delete(&mut self, row_id: RowId) -> Result<(), Error> { + let row = self.row(row_id)?.clone(); + let mut siblings = self.children(row.parent_id); + let at = siblings.iter().position(|id| *id == row_id).unwrap_or(0); + let children = self.children(Some(row_id)); + siblings.splice(at..=at, children.iter().copied()); + self.rows.retain(|candidate| candidate.row_id != row_id); + for (order, sibling) in siblings.into_iter().enumerate() { + if let Ok(sibling) = self.row_mut(sibling) { + sibling.parent_id = row.parent_id; + sibling.sibling_order = order as i64; + } + } + self.expanded.remove(&row_id); + self.normalize(); + Ok(()) + } + + fn row(&self, row_id: RowId) -> Result<&RowStructure, Error> { + self.rows + .iter() + .find(|row| row.row_id == row_id) + .ok_or(Error::UnknownRow(row_id)) + } + + fn row_mut(&mut self, row_id: RowId) -> Result<&mut RowStructure, Error> { + self.rows + .iter_mut() + .find(|row| row.row_id == row_id) + .ok_or(Error::UnknownRow(row_id)) + } + + fn children(&self, parent: Option) -> Vec { + let mut children: Vec<_> = self + .rows + .iter() + .filter(|row| row.parent_id == parent) + .map(|row| (row.sibling_order, row.row_id)) + .collect(); + children.sort_unstable(); + children.into_iter().map(|(_, row_id)| row_id).collect() + } + + fn children_index(&self) -> HashMap, Vec> { + let mut index: HashMap, Vec<(i64, RowId)>> = HashMap::new(); + for row in &self.rows { + index + .entry(row.parent_id) + .or_default() + .push((row.sibling_order, row.row_id)); + } + index + .into_iter() + .map(|(parent, mut children)| { + children.sort_unstable(); + (parent, children.into_iter().map(|(_, id)| id).collect()) + }) + .collect() + } + + fn descendants(&self, row_id: RowId) -> Vec { + let index = self.children_index(); + let mut descendants = Vec::new(); + let mut pending = index.get(&Some(row_id)).cloned().unwrap_or_default(); + while let Some(child) = pending.pop() { + pending.extend(index.get(&Some(child)).into_iter().flatten().copied()); + descendants.push(child); + } + descendants + } + + fn ensure_target(&self, row_id: RowId, target: RowId) -> Result<(), Error> { + self.row(target)?; + if row_id == target || self.descendants(row_id).contains(&target) { + return Err(Error::CannotMoveIntoDescendant); + } + Ok(()) + } + + fn normalize_with_priority(&mut self, row_id: RowId, at: usize) { + let parent = self.row(row_id).ok().and_then(|row| row.parent_id); + let mut siblings = self.children(parent); + siblings.retain(|id| *id != row_id); + siblings.insert(at.min(siblings.len()), row_id); + for (order, sibling) in siblings.into_iter().enumerate() { + if let Ok(row) = self.row_mut(sibling) { + row.sibling_order = order as i64; + } + } + self.normalize(); + } + + fn normalize(&mut self) { + let order: HashMap = self + .children_index() + .into_values() + .flat_map(|children| children.into_iter().zip(0..)) + .collect(); + for row in &mut self.rows { + row.sibling_order = order[&row.row_id]; + } + self.parents.clear(); + for parent in self.rows.iter().filter_map(|row| row.parent_id) { + *self.parents.entry(parent).or_default() += 1; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hierarchy() -> Hierarchy { + let rows = [ + (1, None, 0), + (2, Some(1), 0), + (3, Some(1), 1), + (4, Some(3), 0), + (5, None, 1), + ] + .map(|(row_id, parent_id, sibling_order)| RowStructure { + row_id, + parent_id, + level_key: "item".into(), + sibling_order, + source_path: None, + source_kind: None, + }); + Hierarchy::from_rows(&[1, 2, 3, 4, 5], &rows, "item") + } + + #[test] + fn collapse_and_expand_project_the_tree() { + let mut hierarchy = hierarchy(); + assert_eq!(hierarchy.projection(None), [(1, 0), (5, 0)]); + hierarchy.set_expanded(1, true); + assert_eq!(hierarchy.projection(None), [(1, 0), (2, 1), (3, 1), (5, 0)]); + hierarchy.expand_all(); + assert_eq!( + hierarchy.projection(None), + [(1, 0), (2, 1), (3, 1), (4, 2), (5, 0)] + ); + hierarchy.collapse_all(); + assert_eq!(hierarchy.projection(None), [(1, 0), (5, 0)]); + } + + #[test] + fn restored_expansion_ignores_stale_and_leaf_rows() { + let mut hierarchy = hierarchy(); + hierarchy.restore_expanded(&[1, 2, 3, 99]); + assert_eq!(hierarchy.expanded_rows(), [1, 3]); + assert_eq!( + hierarchy.projection(None), + [(1, 0), (2, 1), (3, 1), (4, 2), (5, 0)] + ); + } + + #[test] + fn a_filter_reveals_matching_descendants_and_their_ancestors() { + let hierarchy = hierarchy(); + assert_eq!( + hierarchy.projection(Some(&HashSet::from([4]))), + [(1, 0), (3, 1), (4, 2)] + ); + } + + #[test] + fn moving_a_subtree_changes_only_its_parent_and_order() { + let mut hierarchy = hierarchy(); + hierarchy.move_row(3, Placement::Root).unwrap(); + hierarchy.move_row(3, Placement::Before(5)).unwrap(); + hierarchy.expand_all(); + assert_eq!( + hierarchy.projection(None), + [(1, 0), (2, 1), (3, 0), (4, 1), (5, 0)] + ); + assert_eq!(hierarchy.row(4).unwrap().parent_id, Some(3)); + assert_eq!(hierarchy.rows().len(), 5); + } + + #[test] + fn moving_a_row_down_lands_beside_its_target() { + for placement in [Placement::After(5), Placement::Before(3)] { + let mut hierarchy = hierarchy(); + hierarchy.move_row(3, Placement::Root).unwrap(); + hierarchy.move_row(1, placement).unwrap(); + assert_eq!(hierarchy.children(None), [5, 1, 3]); + } + } + + #[test] + fn moving_a_parent_into_its_descendant_is_rejected_atomically() { + let mut hierarchy = hierarchy(); + let before = hierarchy.clone(); + assert_eq!( + hierarchy.move_row(1, Placement::ChildOf(4)), + Err(Error::CannotMoveIntoDescendant) + ); + assert_eq!(hierarchy, before); + } + + #[test] + fn indent_and_outdent_use_adjacent_hierarchy_positions() { + let mut hierarchy = hierarchy(); + hierarchy.indent(3).unwrap(); + assert_eq!(hierarchy.row(3).unwrap().parent_id, Some(2)); + hierarchy.outdent(3).unwrap(); + assert_eq!(hierarchy.row(3).unwrap().parent_id, Some(1)); + } + + #[test] + fn delete_promotes_children_into_their_parent_s_place() { + let mut promoted = hierarchy(); + promoted.delete(3).unwrap(); + assert_eq!(promoted.row(4).unwrap().parent_id, Some(1)); + // 4 takes 3's place after 2 rather than joining the end of the list. + assert_eq!(promoted.children(Some(1)), [2, 4]); + } +} diff --git a/crates/table/src/history.rs b/crates/table/src/history.rs index 8d8d0d4b..07c98467 100644 --- a/crates/table/src/history.rs +++ b/crates/table/src/history.rs @@ -56,9 +56,17 @@ pub(crate) enum Step { RowsAdded { at: usize, rows: Vec, + /// Cells an import re-linked on rows already in the table, so it undoes as one step. + cells: Vec<(usize, usize, SharedString, SharedString)>, + before_structure: Vec, + after_structure: Vec, }, /// Ascending by index, which is the order they have to go back in. - RowsRemoved(Vec<(usize, Row)>), + RowsRemoved { + rows: Vec<(usize, Row)>, + before_structure: Vec, + after_structure: Vec, + }, ColumnAdded { at: usize, col: Col, @@ -78,6 +86,10 @@ pub(crate) enum Step { }, /// Several steps taken as one, in the order they were made — a restore. Batch(Vec), + Hierarchy { + before: Vec, + after: Vec, + }, } impl Step { @@ -85,11 +97,12 @@ impl Step { fn is_empty(&self) -> bool { match self { Step::Cells(cells) => cells.is_empty(), - Step::RowsAdded { rows, .. } => rows.is_empty(), - Step::RowsRemoved(rows) => rows.is_empty(), + Step::RowsAdded { rows, cells, .. } => rows.is_empty() && cells.is_empty(), + Step::RowsRemoved { rows, .. } => rows.is_empty(), Step::Renamed { before, after, .. } => before == after, Step::ColumnMoved { from, to } => from == to, Step::Batch(steps) => steps.iter().all(Step::is_empty), + Step::Hierarchy { before, after } => before == after, _ => false, } } @@ -121,6 +134,11 @@ impl History { } } + /// The step just recorded, for a command built from two edits that must undo as one. + pub(crate) fn last_mut(&mut self) -> Option<&mut Step> { + self.done.last_mut() + } + /// The last step, to be replayed backwards. `None` when there is nothing left to undo. pub(crate) fn undo(&mut self) -> Option { let step = self.done.pop()?; @@ -214,7 +232,11 @@ mod tests { fn an_empty_step_is_not_recorded() { let mut h = History::default(); h.push(Step::Cells(Vec::new())); - h.push(Step::RowsRemoved(Vec::new())); + h.push(Step::RowsRemoved { + rows: Vec::new(), + before_structure: Vec::new(), + after_structure: Vec::new(), + }); h.push(Step::Renamed { col: 0, before: "A".into(), @@ -228,20 +250,22 @@ mod tests { #[test] fn a_structural_step_undoes_and_redoes() { let mut h = History::default(); - let removed = || { - Step::RowsRemoved(vec![( + let removed = || Step::RowsRemoved { + rows: vec![( 2, Row { id: 3, cells: vec!["gone".into()], image: None, }, - )]) + )], + before_structure: Vec::new(), + after_structure: Vec::new(), }; h.push(edit(&[(0, 0, "a", "A")])); h.push(removed()); - let Some(Step::RowsRemoved(rows)) = h.undo() else { + let Some(Step::RowsRemoved { rows, .. }) = h.undo() else { panic!("the structural step comes back first"); }; assert_eq!(rows[0].0, 2); @@ -250,6 +274,6 @@ mod tests { // ...and the cell edit under it is still there, unshifted. assert_eq!(befores(&h.undo().unwrap()), vec![(0, 0, "a".into())]); assert!(matches!(h.redo(), Some(Step::Cells(_)))); - assert!(matches!(h.redo(), Some(Step::RowsRemoved(_)))); + assert!(matches!(h.redo(), Some(Step::RowsRemoved { .. }))); } } diff --git a/crates/table/src/lib.rs b/crates/table/src/lib.rs index 141d14b0..965ebb41 100644 --- a/crates/table/src/lib.rs +++ b/crates/table/src/lib.rs @@ -15,6 +15,7 @@ pub mod editor; pub mod file_links; mod filter; pub mod floating; +mod hierarchy; mod history; mod note; mod panel; @@ -29,11 +30,16 @@ pub use editor::editor_box; /// growing a second, quietly diverging copy. pub use note::{Target as MenuTarget, menu as context_menu}; pub use panel::{ - Clear, Copy, Cut, DeleteColumn, DeleteRow, Deselect, DuplicateRow, EditCell, GRID_CONTEXT, - InsertColumnLeft, InsertColumnRight, InsertNote, InsertRowAbove, InsertRowBelow, Paste, Redo, + Clear, CollapseAll, Copy, Cut, DeleteColumn, DeleteRow, DeleteSubtree, Deselect, DuplicateRow, + EditCell, ExpandAll, GRID_CONTEXT, ImportFiles, IndentRow, InsertColumnLeft, InsertColumnRight, + InsertNote, InsertRowAbove, InsertRowBelow, OutdentRow, Paste, Redo, RelinkMissingFiles, RenameColumn, Replace, Search, TablePanel, Undo, UnfreezeColumns, }; +/// Global command handle for import entry points outside the centre table, such as Details. +pub struct TablePanelHandle(pub WeakEntity); +impl Global for TablePanelHandle {} + /// Settings key (in either scope) for the alternating-row-stripe toggle. pub const TABLE_STRIPES_KEY: &str = "table_stripes"; @@ -259,6 +265,7 @@ fn settle( } } TablePanel::persist_columns(state, cx); + sync_project_columns(state, None, cx); settings::dirty::mark(settings::dirty::PROJECT_DATA, cx); revalidate_now(cx); autosave(cx); @@ -393,6 +400,100 @@ pub enum Structural { }, } +pub enum Arrangement { + Indent(usize), + /// Wrap these source rows in a new parent. + Group(Vec), + Outdent(usize), + Reparent { + row: usize, + parent: Option, + }, + DeleteSubtree(usize), + Move { + row: usize, + target: usize, + placement: RowPlacement, + }, +} + +#[derive(Clone, Copy)] +pub enum RowPlacement { + Before, + Child, + After, +} + +pub fn arrange(op: Arrangement, cx: &mut App) { + let Some(state) = cx + .try_global::() + .and_then(|handle| handle.0.upgrade()) + else { + return; + }; + if let Arrangement::DeleteSubtree(row) = op { + let rows = state.read(cx).delegate().subtree_sources(row); + structural(Structural::DeleteRows(rows), cx); + return; + } + let result = state.update(cx, |state, cx| { + let result = match op { + Arrangement::Indent(row) => state.delegate_mut().indent_row(row), + Arrangement::Group(rows) => state.delegate_mut().group_rows(&rows), + Arrangement::Outdent(row) => state.delegate_mut().outdent_row(row), + Arrangement::Reparent { row, parent } => state.delegate_mut().reparent_row(row, parent), + Arrangement::Move { + row, + target, + placement, + } => state + .delegate_mut() + .move_row_relative(row, target, placement), + Arrangement::DeleteSubtree(_) => unreachable!(), + }; + if result.is_ok() { + state.refresh(cx); + cx.emit(delegate::TableChanged); + cx.notify(); + } + result + }); + if let Err(error) = result { + log::warn!("Could not change archival hierarchy: {error:?}"); + return; + } + settings::dirty::mark(settings::dirty::PROJECT_DATA, cx); + autosave(cx); +} + +pub(crate) fn persist_expanded(rows: &[settings::project::RowId], cx: &mut App) { + if !cx.has_global::() { + return; + } + settings::project::CurrentProject::set_text( + settings::description::HIERARCHY_EXPANDED_KEY, + serde_json::to_string(rows) + .unwrap_or_else(|_| "[]".into()) + .into(), + cx, + ); +} + +/// Hand the grid's column names to the open project, which is what every other reader of the +/// column list looks at. +fn sync_project_columns( + state: &Entity>, + renamed: Option<&(SharedString, SharedString)>, + cx: &mut App, +) { + let delegate = state.read(cx).delegate(); + let headers = (0..delegate.column_count()) + .map(|col| delegate.column_name(col).to_string()) + .collect(); + let renamed = renamed.map(|(before, after)| (before.as_ref(), after.as_ref())); + settings::project::CurrentProject::set_columns(headers, renamed, cx); +} + /// Apply a shape change to the table and everything keyed off the shape. /// /// Validation runs *now*, not on `TablePanel`'s debounce: after a row shift the open diagnostics @@ -464,6 +565,13 @@ pub fn structural(op: Structural, cx: &mut App) { cx, ); } + if matches!( + op, + Structural::InsertColumn { .. } | Structural::DeleteColumn { .. } + ) || renamed.is_some() + { + sync_project_columns(&state, renamed.as_ref(), cx); + } if let Some((before, after)) = renamed { follow_rename(&before, &after, cx); } @@ -638,24 +746,39 @@ pub fn save_now(cx: &mut App) { return; }; let started = std::time::Instant::now(); - let (headers, row_ids, rows) = state.read(cx).delegate().dataset_snapshot(); let author = settings::history::author(cx); - let history: Vec<_> = state - .read(cx) - .delegate() - .unsaved_history() - .iter() - .map(|entry| settings::history::Entry { - author: author.clone(), - ..entry.clone() - }) - .collect(); - match settings::project::save_dataset(&file, &headers, &row_ids, &rows, &history) { + let (headers, row_ids, rows, structure, history) = { + let delegate = state.read(cx).delegate(); + let (headers, row_ids, rows) = delegate.dataset_snapshot(); + let structure = delegate.row_structure().to_vec(); + let history = delegate + .unsaved_history() + .iter() + .map(|entry| settings::history::Entry { + author: author.clone(), + ..entry.clone() + }) + .collect::>(); + (headers, row_ids, rows, structure, history) + }; + match settings::project::save_dataset( + &file, + &headers, + &row_ids, + &rows, + Some(&structure), + &history, + ) { Ok(()) => { state.update(cx, |state, _| { state.delegate_mut().history_saved(history.len()) }); - log::debug!("saved {} rows in {:?}", rows.len(), started.elapsed()); + log::debug!( + "saved {} rows and {} arranged components in {:?}", + rows.len(), + structure.len(), + started.elapsed() + ); settings::dirty::clear(settings::dirty::PROJECT_DATA, cx); } Err(err) => log::error!("failed to save project data: {err}"), diff --git a/crates/table/src/note.rs b/crates/table/src/note.rs index 5d2b183b..c42dc194 100644 --- a/crates/table/src/note.rs +++ b/crates/table/src/note.rs @@ -229,13 +229,29 @@ fn selection_items(menu: PopupMenu, rows: &[usize], files: Vec) -> Popu /// The row and column commands every target that has a row or a column offers. Built here rather /// than three times over, because the cell menu is the row menu and the column menu at once. -fn structural_items(menu: PopupMenu, rows: Option<&[usize]>, col: Option) -> PopupMenu { +/// Whether the menu's target rows hold an arrangement — the only case where deleting has two +/// meanings worth offering. +fn has_children(delegate: &QrateTableDelegate, rows: &[usize]) -> bool { + rows.iter().any(|&row| delegate.row_has_children(row)) +} + +fn structural_items( + menu: PopupMenu, + rows: Option<&[usize]>, + parents: bool, + col: Option, +) -> PopupMenu { use crate::Structural; let menu = match rows { Some(rows) => { - let (above, below, dup, delete) = - (rows[0], rows[rows.len() - 1], rows[0], rows.to_vec()); + let (above, below, dup, delete, group) = ( + rows[0], + rows[rows.len() - 1], + rows[0], + rows.to_vec(), + rows.to_vec(), + ); menu.item( PopupMenuItem::new("Insert row above").on_click(move |_, _, cx| { crate::structural(Structural::InsertRow { at: above }, cx) @@ -252,10 +268,43 @@ fn structural_items(menu: PopupMenu, rows: Option<&[usize]>, col: Option) }), ) .item( - PopupMenuItem::new(rows_label("Delete", rows)).on_click(move |_, _, cx| { + PopupMenuItem::new(rows_label("Group", rows)).on_click(move |_, _, cx| { + crate::arrange(crate::Arrangement::Group(group.clone()), cx) + }), + ) + .item( + PopupMenuItem::new("Indent row") + .on_click(move |_, _, cx| crate::arrange(crate::Arrangement::Indent(dup), cx)), + ) + .item( + PopupMenuItem::new("Outdent row") + .on_click(move |_, _, cx| crate::arrange(crate::Arrangement::Outdent(dup), cx)), + ) + .item( + // A leaf has nothing to promote or carry with it, so it gets the one plain + // command the two arrangement-aware ones would only dress up. + PopupMenuItem::new(match parents { + true => rows_label("Delete and promote children", rows), + false => rows_label("Delete", rows), + }) + .on_click(move |_, _, cx| { crate::structural(Structural::DeleteRows(delete.clone()), cx) }), ) + .when(parents, |menu| { + menu.item(PopupMenuItem::new("Delete row and descendants").on_click( + move |_, _, cx| crate::arrange(crate::Arrangement::DeleteSubtree(dup), cx), + )) + }) + .separator() + .item( + PopupMenuItem::new("Expand all groups") + .on_click(move |_, _, cx| cx.dispatch_action(&crate::ExpandAll)), + ) + .item( + PopupMenuItem::new("Collapse all groups") + .on_click(move |_, _, cx| cx.dispatch_action(&crate::CollapseAll)), + ) } None => menu, }; @@ -403,57 +452,67 @@ pub fn menu( ) .separator(); let rows = target_rows(table.read(cx).delegate(), row); - structural_items(menu, Some(&rows), Some(col)) - .separator() - .item( - PopupMenuItem::new("Clear contents") - .action(Box::new(crate::Clear)) - .on_click(move |_, _, cx| { - crate::write_cell(row, col, SharedString::default(), Origin::Clear, cx) - }), - ) - .separator() - .item( - PopupMenuItem::new("Edit cell") - .action(Box::new(crate::EditCell)) - .on_click(move |_, window, cx| { - edit_table.update(cx, |state, cx| { - crate::editing::start(state.delegate_mut(), row, col, window, cx); - cx.notify(); - }); - }), - ) - .item( - PopupMenuItem::new("Filter by this value").on_click(move |_, _, cx| { - filter_table.update(cx, |state, cx| { - state.delegate_mut().keep_only_value(col, &value); - cx.emit(TableChanged); + structural_items( + menu, + Some(&rows), + has_children(table.read(cx).delegate(), &rows), + Some(col), + ) + .separator() + .item( + PopupMenuItem::new("Clear contents") + .action(Box::new(crate::Clear)) + .on_click(move |_, _, cx| { + crate::write_cell(row, col, SharedString::default(), Origin::Clear, cx) + }), + ) + .separator() + .item( + PopupMenuItem::new("Edit cell") + .action(Box::new(crate::EditCell)) + .on_click(move |_, window, cx| { + edit_table.update(cx, |state, cx| { + crate::editing::start(state.delegate_mut(), row, col, window, cx); cx.notify(); }); }), - ) - .when_some( - location.row_id.zip(location.column.clone()), - |menu, (row, column)| { - menu.item(PopupMenuItem::new("Show edit history").on_click( - move |_, _, cx| { - cx.set_global(settings::history::ShowCellHistory { - row, - column: column.to_string(), - }) - }, - )) - }, - ) - .separator() + ) + .item( + PopupMenuItem::new("Filter by this value").on_click(move |_, _, cx| { + filter_table.update(cx, |state, cx| { + state.delegate_mut().keep_only_value(col, &value); + cx.emit(TableChanged); + cx.notify(); + }); + }), + ) + .when_some( + location.row_id.zip(location.column.clone()), + |menu, (row, column)| { + menu.item( + PopupMenuItem::new("Show edit history").on_click(move |_, _, cx| { + cx.set_global(settings::history::ShowCellHistory { + row, + column: column.to_string(), + }) + }), + ) + }, + ) + .separator() } Target::Row(row) => { let rows = target_rows(table.read(cx).delegate(), row); - structural_items(menu, Some(&rows), None) - .item(PopupMenuItem::new("Clear row").on_click(move |_, _, cx| { - crate::write_cells(crate::blank_row(row, cx), Origin::Clear, cx) - })) - .separator() + structural_items( + menu, + Some(&rows), + has_children(table.read(cx).delegate(), &rows), + None, + ) + .item(PopupMenuItem::new("Clear row").on_click(move |_, _, cx| { + crate::write_cells(crate::blank_row(row, cx), Origin::Clear, cx) + })) + .separator() } // Built-in column controls come first; plugin entries are appended without running plugin // code while the synchronous menu is built. @@ -472,7 +531,7 @@ pub fn menu( let stored = settings::columns::get(&key, cx).plugins; let variant_review = settings::columns::get(&key, cx).variant_review; let (freeze_table, rename_table) = (table.clone(), table.clone()); - let menu = structural_items(menu, None, Some(col)) + let menu = structural_items(menu, None, false, Some(col)) .item( PopupMenuItem::new("Rename column…").on_click(move |_, window, cx| { rename_table.update(cx, |state, cx| { diff --git a/crates/table/src/panel.rs b/crates/table/src/panel.rs index 42706ab1..570c7f58 100644 --- a/crates/table/src/panel.rs +++ b/crates/table/src/panel.rs @@ -63,7 +63,14 @@ actions!( InsertColumnLeft, InsertColumnRight, DeleteColumn, - RenameColumn + RenameColumn, + ExpandAll, + CollapseAll, + IndentRow, + OutdentRow, + DeleteSubtree, + ImportFiles, + RelinkMissingFiles ] ); @@ -137,6 +144,31 @@ impl TablePanel { &project.data.row_ids, &project.data.rows, ); + let structure = + settings::project::read_row_structure(&project.file).unwrap_or_else(|error| { + log::warn!( + "Could not read row structure from {:?}: {error}", + project.file + ); + Vec::new() + }); + let description = + settings::description::DescriptionConfig::from_values(&project.data.values); + delegate.set_structure(&structure, &description.file_level_key); + let expanded: Vec = project + .data + .values + .get(settings::description::HIERARCHY_EXPANDED_KEY) + .and_then(|value| serde_json::from_str(&value.text()).ok()) + .unwrap_or_default(); + delegate.restore_expanded(&expanded); + log::info!( + "opened {} with {} rows, {} arranged components, {} profile", + project.file.display(), + project.data.rows.len(), + structure.len(), + description.profile.key() + ); Self::apply_saved_layout(&mut delegate, &project.file); delegate.set_image_paths(Self::resolve_images(&project.data)); loaded_project = Some(project.file.clone()); @@ -355,9 +387,32 @@ impl TablePanel { project.data.rows.clone(), ); let image_paths = Self::resolve_images(&project.data); + let structure = + settings::project::read_row_structure(&file).unwrap_or_else(|error| { + log::warn!("Could not read row structure from {file:?}: {error}"); + Vec::new() + }); + let description = + settings::description::DescriptionConfig::from_values(&project.data.values); + let expanded: Vec = project + .data + .values + .get(settings::description::HIERARCHY_EXPANDED_KEY) + .and_then(|value| serde_json::from_str(&value.text()).ok()) + .unwrap_or_default(); this.loaded_project = Some(file.clone()); + log::info!( + "loaded {} rows, {} arranged components, {} profile", + rows.len(), + structure.len(), + description.profile.key() + ); this.state.update(cx, |state, cx| { state.delegate_mut().set_data(&headers, &row_ids, &rows); + state + .delegate_mut() + .set_structure(&structure, &description.file_level_key); + state.delegate_mut().restore_expanded(&expanded); Self::apply_saved_layout(state.delegate_mut(), &file); state.delegate_mut().set_image_paths(image_paths); apply_settings(state.delegate_mut(), cx); @@ -429,7 +484,7 @@ impl TablePanel { } }); - Self { + let panel = Self { focus_handle: cx.focus_handle(), state, loaded_project, @@ -457,7 +512,9 @@ impl TablePanel { _replace_sub, _autosave_task: None, _revalidate_task: None, - } + }; + cx.set_global(crate::TablePanelHandle(cx.entity().downgrade())); + panel } /// Re-run the validators once the edits stop, rather than inside the commit. @@ -473,6 +530,257 @@ impl TablePanel { })); } + pub fn import_external_paths( + &mut self, + paths: Vec, + window: &mut Window, + cx: &mut Context, + ) { + let description = cx + .try_global::() + .map(|project| { + settings::description::DescriptionConfig::from_values(&project.data.values) + }) + .unwrap_or_else(|| settings::description::DescriptionProfile::Rad.defaults()); + let folder_level = description.folder_level_key; + let file_level = description.file_level_key; + let task = cx.background_executor().spawn(async move { + file_ingest::plan_paths( + &paths, + &file_ingest::PlanOptions { + recursive: true, + include_root: true, + folder_level_key: &folder_level, + file_level_key: &file_level, + }, + ) + }); + let policy = cx + .try_global::() + .and_then(|project| { + project + .data + .values + .get(settings::project::IMPORT_DUPLICATE_POLICY_KEY) + }) + .map(|value| file_ingest::duplicates::DuplicatePolicy::parse(&value.text())) + .unwrap_or_default(); + cx.spawn_in(window, async move |this, cx| { + let plan = match task.await { + Ok(plan) => plan, + Err(error) => { + log::warn!( + "Nothing was imported: the dropped paths could not be read ({error:?})" + ); + return; + } + }; + for warning in &plan.warnings { + log::warn!("Left a dropped path out of the import: {warning:?}"); + } + this.update_in(cx, |this, window, cx| { + let project = cx.global::(); + let files_root = project + .data + .values + .get(settings::project::FILES_FOLDER_KEY) + .map(|folder| std::path::PathBuf::from(folder.text().as_ref())); + let (title_name, file_name) = project.data.columns.iter().fold( + (None, None), + |(title, file), column| { + match settings::columns::ColumnType::from_declared(&column.data_type) { + settings::columns::ColumnType::Title => { + (Some(column.name.clone()), file) + } + settings::columns::ColumnType::Filename => { + (title, Some(column.name.clone())) + } + _ => (title, file), + } + }, + ); + let (title_col, file_col) = { + let delegate = this.state.read(cx).delegate(); + ( + title_name.as_deref().and_then(|name| delegate.data_col(name)), + file_name.as_deref().and_then(|name| delegate.data_col(name)), + ) + }; + let existing = this + .state + .read(cx) + .delegate() + .existing_components(file_col, files_root.as_deref()); + let resolved = file_ingest::duplicates::resolve( + plan, + &existing, + settings::filenames::keys, + policy, + ); + + let files = resolved + .plan + .components + .iter() + .filter(|component| component.kind == file_ingest::EntryKind::File) + .count(); + let folders = resolved.plan.components.len() - files; + let duplicates = resolved.duplicates(); + let detail = format!( + "Add {} component{} ({} file{}, {} folder{})?{}{}{}", + resolved.plan.components.len(), + if resolved.plan.components.len() == 1 { "" } else { "s" }, + files, + if files == 1 { "" } else { "s" }, + folders, + if folders == 1 { "" } else { "s" }, + if duplicates == 0 { + String::new() + } else { + format!(" {duplicates} are already in this project.") + }, + if resolved.ambiguous() == 0 { + String::new() + } else { + format!( + " {} match several rows and will be added as new rows.", + resolved.ambiguous() + ) + }, + if resolved.plan.warnings.is_empty() { + String::new() + } else { + format!( + " {} path warning(s) will be skipped.", + resolved.plan.warnings.len() + ) + } + ); + // Only a drop that actually repeats material asks what to do about it. + let choices: &[&str] = match duplicates { + 0 => &["Import", "Cancel"], + _ => &["Skip duplicates", "Update existing", "Add all as new", "Cancel"], + }; + let answer = window.prompt( + PromptLevel::Info, + "Import dropped files", + Some(&detail), + choices, + cx, + ); + cx.spawn_in(window, async move |this, cx| { + let chosen = answer.await.unwrap_or(usize::MAX); + let chosen = match (duplicates, chosen) { + (0, 0) => policy, + (_, 0) => file_ingest::duplicates::DuplicatePolicy::Skip, + (_, 1) => file_ingest::duplicates::DuplicatePolicy::Update, + (_, 2) => file_ingest::duplicates::DuplicatePolicy::AddAsNew, + _ => return, + }; + this.update(cx, |this, cx| { + // Re-resolving is what makes the buttons mean what they say. + let resolved = match chosen == policy { + true => resolved, + false => file_ingest::duplicates::resolve( + resolved.plan, + &existing, + settings::filenames::keys, + chosen, + ), + }; + this.state.update(cx, |state, cx| { + let added = state.delegate_mut().append_components( + &resolved, + title_col, + file_col, + None, + files_root.as_deref(), + ); + log::info!( + "imported {added} dropped component(s) as {}, reusing {} already in the project", + chosen.key(), + resolved.duplicates() + ); + state.refresh(cx); + cx.emit(TableChanged); + cx.notify(); + }); + let row_ids = this.state.read(cx).delegate().row_ids().to_vec(); + diagnostics::Diagnostics::align_note_rows( + diagnostics::DATASET_MAIN, + &row_ids, + settings::history::Origin::Structure, + cx, + ); + settings::dirty::mark(settings::dirty::PROJECT_DATA, cx); + this.schedule_revalidate(cx); + this.schedule_autosave(cx); + }) + .ok(); + }) + .detach(); + }) + .ok(); + }) + .detach(); + } + + pub fn choose_import_paths(&mut self, window: &mut Window, cx: &mut Context) { + let receiver = cx.prompt_for_paths(PathPromptOptions { + files: true, + directories: true, + multiple: true, + prompt: Some("Choose files or folders to import".into()), + }); + cx.spawn_in(window, async move |this, cx| { + if let Ok(Ok(Some(paths))) = receiver.await + && !paths.is_empty() + { + this.update_in(cx, |this, window, cx| { + this.import_external_paths(paths, window, cx) + }) + .ok(); + } + }) + .detach(); + } + + pub fn choose_files_root(&mut self, window: &mut Window, cx: &mut Context) { + let receiver = cx.prompt_for_paths(PathPromptOptions { + files: false, + directories: true, + multiple: false, + prompt: Some("Choose the folder that contains this project's files".into()), + }); + cx.spawn_in(window, async move |this, cx| { + if let Ok(Ok(Some(paths))) = receiver.await + && let Some(folder) = paths.first() + { + let folder = folder.to_string_lossy().into_owned(); + this.update(cx, |this, cx| { + settings::project::CurrentProject::set_text( + settings::project::FILES_FOLDER_KEY, + folder.into(), + cx, + ); + let image_paths = cx + .try_global::() + .map(|project| Self::resolve_images(&project.data)) + .unwrap_or_default(); + this.state.update(cx, |state, cx| { + state.delegate_mut().set_image_paths(image_paths); + state.refresh(cx); + cx.notify(); + }); + crate::revalidate_now(cx); + cx.notify(); + }) + .ok(); + } + }) + .detach(); + } + /// React to a committed cell edit per the Autosave setting: write immediately, buffer behind a /// short debounce (the default), or leave it for Ctrl+S / quit. The default and any unset/ /// unrecognized value both mean "timed". @@ -1314,6 +1622,10 @@ impl Render for TablePanel { .id("table-panel") .role(Role::Group) .aria_label("Table") + .drag_over::(|style, _, _, cx| style.bg(cx.theme().secondary_hover)) + .on_drop(cx.listener(|this, paths: &ExternalPaths, window, cx| { + this.import_external_paths(paths.paths().to_vec(), window, cx) + })) // Search, Replace and the find bar's own Escape are handled by `ViewsPanel`, which draws // the bar above every view. An action stops propagating by default, so declining an // Escape that is not ours has to be explicit. @@ -1351,6 +1663,47 @@ impl Render for TablePanel { .on_action(cx.listener(|this, _: &UnfreezeColumns, _, cx| { crate::set_frozen_columns(&this.state.clone(), 0, cx) })) + .on_action(cx.listener(|this, _: &ExpandAll, _, cx| { + let expanded = this.state.update(cx, |state, cx| { + state.delegate_mut().expand_all(); + let expanded = state.delegate().expanded_rows(); + state.refresh(cx); + cx.emit(TableChanged); + expanded + }); + crate::persist_expanded(&expanded, cx); + })) + .on_action(cx.listener(|this, _: &CollapseAll, _, cx| { + this.state.update(cx, |state, cx| { + state.delegate_mut().collapse_all(); + state.refresh(cx); + cx.emit(TableChanged); + }); + crate::persist_expanded(&[], cx); + })) + .on_action(cx.listener(|this, _: &IndentRow, _, cx| { + if let Some((rows, _)) = this.structural_target(cx) { + crate::arrange(crate::Arrangement::Indent(rows[0]), cx); + } + })) + .on_action(cx.listener(|this, _: &OutdentRow, _, cx| { + if let Some((rows, _)) = this.structural_target(cx) { + crate::arrange(crate::Arrangement::Outdent(rows[0]), cx); + } + })) + .on_action(cx.listener(|this, _: &DeleteSubtree, _, cx| { + if let Some((rows, _)) = this.structural_target(cx) { + crate::arrange(crate::Arrangement::DeleteSubtree(rows[0]), cx); + } + })) + .on_action( + cx.listener(|this, _: &ImportFiles, window, cx| { + this.choose_import_paths(window, cx) + }), + ) + .on_action(cx.listener(|this, _: &RelinkMissingFiles, window, cx| { + this.choose_files_root(window, cx) + })) .on_action(cx.listener(|this, _: &InsertRowAbove, _, cx| { this.structural(|rows, _| crate::Structural::InsertRow { at: rows[0] }, cx) })) diff --git a/crates/table/src/photos.rs b/crates/table/src/photos.rs index 357d0907..2425ca57 100644 --- a/crates/table/src/photos.rs +++ b/crates/table/src/photos.rs @@ -12,47 +12,33 @@ use settings::columns::ColumnType; use std::collections::HashMap; -use std::fs; use std::path::{Path, PathBuf}; -/// Filenames a recursive walk should ignore — OS cruft, not collection data. -fn is_ignored(name: &str) -> bool { - name.starts_with("._") || name.eq_ignore_ascii_case(".ds_store") -} - -/// Every file under `folder`, found recursively, keyed by lowercased filename (with extension). -/// A depth-first stack walk rather than `walkdir` — the whole point of a recursive listing here -/// is a handful of nesting levels, not arbitrary directory trees, so std's `read_dir` is enough. +/// Every file under `folder`, found recursively, keyed by relative path and lowercased filename. +/// The shared inventory preserves paths for duplicate basenames; the basename keys retain the +/// tolerant matching used by existing projects. fn index_files(folder: &Path) -> HashMap { let mut index = HashMap::new(); - let mut stack = vec![folder.to_path_buf()]; - while let Some(dir) = stack.pop() { - let Ok(entries) = fs::read_dir(&dir) else { + let Ok(inventory) = file_ingest::scan(folder, true) else { + return index; + }; + for entry in inventory.files() { + let Some(name) = entry.path.file_name().and_then(|name| name.to_str()) else { continue; }; - for entry in entries.filter_map(|e| e.ok()) { - let path = entry.path(); - let Some(name) = path.file_name().and_then(|n| n.to_str()) else { - continue; - }; - if is_ignored(name) { - continue; - } - if path.is_dir() { - stack.push(path); - } else { - // Two batches can hold the same filename. Keeping the first by path rather than - // the first walked means the same row shows the same picture on every launch. - index - .entry(name.to_lowercase()) - .and_modify(|kept| { - if path < *kept { - *kept = path.clone(); - } - }) - .or_insert(path); - } - } + let path = entry.path.clone(); + let relative = file_ingest::normalized_path(&entry.relative_path).to_lowercase(); + index.insert(relative, path.clone()); + // Two batches can hold the same filename. Keeping the first by path rather than the first + // walked means the same row shows the same picture on every launch. + index + .entry(name.to_lowercase()) + .and_modify(|kept| { + if path < *kept { + *kept = path.clone(); + } + }) + .or_insert(path); } index } @@ -87,10 +73,14 @@ impl PhotoIndex { Self { by_key } } - pub(crate) fn resolve_cell(&self, cell: &str) -> Option<&PathBuf> { + pub(crate) fn resolve_cell(&self, cell: &str) -> Option { + let direct = PathBuf::from(cell); + if direct.is_absolute() && direct.is_file() { + return Some(direct); + } settings::filenames::lookup_keys(cell) .iter() - .find_map(|key| self.by_key.get(key)) + .find_map(|key| self.by_key.get(key).cloned()) } /// The image path for `row`'s cells, if any cell names a file this index found. Checks the @@ -118,9 +108,9 @@ impl PhotoIndex { .and_then(|ix| row.get(ix)) .and_then(|c| self.resolve_cell(c)) { - return Some(hit.clone()); + return Some(hit); } - row.iter().find_map(|c| self.resolve_cell(c).cloned()) + row.iter().find_map(|c| self.resolve_cell(c)) } } @@ -177,6 +167,7 @@ pub(crate) fn refresh( #[cfg(test)] mod tests { use super::*; + use std::fs; use std::io::Write; fn tempdir(name: &str) -> PathBuf { @@ -233,6 +224,23 @@ mod tests { ); } + #[test] + fn a_relative_path_disambiguates_duplicate_filenames() { + let dir = tempdir("duplicate-basename"); + touch(&dir.join("photographs").join("001.jpg")); + touch(&dir.join("documents").join("001.jpg")); + + let index = PhotoIndex::build(dir.to_str().unwrap()); + assert_eq!( + index.resolve_cell("photographs/001.jpg"), + Some(dir.join("photographs").join("001.jpg")) + ); + assert_eq!( + index.resolve_cell("documents/001.jpg"), + Some(dir.join("documents").join("001.jpg")) + ); + } + #[test] fn an_item_id_resolves_to_the_first_of_its_parts() { // The row names the item; the folder holds its numbered parts. Whichever order the walk @@ -244,7 +252,7 @@ mod tests { let index = PhotoIndex::build(dir.to_str().unwrap()); assert_eq!( index.resolve_cell("2020_04_001"), - Some(&dir.join("2020_04_001_001.jpg")) + Some(dir.join("2020_04_001_001.jpg")) ); } diff --git a/crates/table/src/row_index.rs b/crates/table/src/row_index.rs index 43122662..6a6c4c03 100644 --- a/crates/table/src/row_index.rs +++ b/crates/table/src/row_index.rs @@ -1,10 +1,13 @@ use std::sync::OnceLock; use gpui::prelude::FluentBuilder as _; -use gpui::{Context, FontWeight, IntoElement, SharedString, div, prelude::*, px}; +use gpui::{ + Context, FontWeight, InteractiveElement as _, IntoElement, SharedString, div, prelude::*, px, +}; use gpui_component::menu::ContextMenuExt as _; use gpui_component::{ - ActiveTheme, + ActiveTheme, IconName, Sizable as _, + button::{Button, ButtonVariants as _}, table::{Column, TableState}, }; @@ -15,7 +18,25 @@ use crate::note::{self, Target}; pub(crate) const COL_IX: usize = 0; -const WIDTH: f32 = 48.; +const WIDTH: f32 = 128.; + +#[derive(Clone)] +struct RowDrag(usize); + +struct RowDragPreview(usize); + +impl gpui::Render for RowDragPreview { + fn render(&mut self, _: &mut gpui::Window, cx: &mut gpui::Context) -> impl IntoElement { + div() + .px_2() + .py_1() + .rounded_md() + .bg(cx.theme().background) + .border_1() + .border_color(cx.theme().primary) + .child(format!("Move component {}", self.0 + 1)) + } +} pub(crate) fn column() -> Column { static COLUMN: OnceLock = OnceLock::new(); @@ -33,6 +54,7 @@ pub(crate) fn column() -> Column { pub(crate) fn render_td( delegate: &QrateTableDelegate, + view_ix: usize, row_ix: usize, highlighted: bool, cx: &mut Context>, @@ -45,16 +67,89 @@ pub(crate) fn render_td( let location = delegate.location(Some(row_ix), None); let worst = Diagnostics::worst_at(&location.dataset, Some(row_ix), None, cx); let tip = note::tooltip_text(&location, cx); + let depth = delegate.row_depth(view_ix); + let children = delegate.row_child_count(row_ix); + let expanded = delegate.row_expanded(row_ix); div() .id(("row-note", row_ix)) .size_full() + .relative() .flex() .items_center() - .justify_center() + .pl(px(6.)) + .gap_1() + .on_drag(RowDrag(row_ix), move |drag, _, _, cx| { + cx.new(|_| RowDragPreview(drag.0)) + }) + .child( + div().absolute().inset_0().flex().flex_col().children( + [ + crate::RowPlacement::Before, + crate::RowPlacement::Child, + crate::RowPlacement::After, + ] + .map(|placement| { + div() + .flex_1() + .drag_over::(|style, _, _, cx| style.bg(cx.theme().table_active)) + .on_drop(move |drag: &RowDrag, window, cx| { + let source = drag.0; + window.defer(cx, move |_, cx| { + crate::arrange( + crate::Arrangement::Move { + row: source, + target: row_ix, + placement, + }, + cx, + ); + }); + }) + }), + ), + ) .bg(bg) + // The Problems panel's nesting: a tint under anything inside a group, and one guide line + // per level instead of bare indentation. + .when(depth > 0 && !highlighted, |d| { + d.bg(cx.theme().muted.opacity(0.35)) + }) + .children((0..depth.min(4)).map(|_| { + div() + .w(px(12.)) + .h_full() + .flex_shrink_0() + .border_l_1() + .border_color(cx.theme().border) + })) .text_color(fg) .when(highlighted, |d| d.font_weight(FontWeight::SEMIBOLD)) + .when(children > 0, |d| { + d.child( + Button::new(("row-chevron", row_ix)) + .text() + .small() + .icon(match expanded { + true => IconName::ChevronDown, + false => IconName::ChevronRight, + }) + .label(format!("({children})")) + .accessibility_label(format!( + "{} component {}", + if expanded { "Collapse" } else { "Expand" }, + row_ix + 1 + )) + .on_click(cx.listener(move |state, _, _, cx| { + cx.stop_propagation(); + state.delegate_mut().toggle_expanded(row_ix); + let expanded = state.delegate().expanded_rows(); + state.refresh(cx); + crate::persist_expanded(&expanded, cx); + cx.notify(); + })), + ) + }) .child(SharedString::from((row_ix + 1).to_string())) .when_some(worst, |d, severity| d.child(note::marker(severity, cx))) .when_some(tip, |d, text| { diff --git a/crates/workspace/src/panels/details.rs b/crates/workspace/src/panels/details.rs index 5e99255d..22059213 100644 --- a/crates/workspace/src/panels/details.rs +++ b/crates/workspace/src/panels/details.rs @@ -19,7 +19,7 @@ use gpui_component::{ }; use preview::{can_preview, thumb}; use settings::history::{Change, EntryId, Listed, Origin}; -use table::{QrateTableDelegate, TableChanged, TableStateHandle}; +use table::{QrateTableDelegate, TableChanged, TablePanelHandle, TableStateHandle}; use crate::BottomDockCrop; use crate::panel_registry::PanelMeta; @@ -1133,6 +1133,19 @@ impl Render for DetailsPanel { // is empty — the multi-select gesture is the one thing here nobody discovers by luck. return div() .size_full() + .drag_over::(|style, _, _, cx| { + style.bg(cx.theme().secondary_hover) + }) + .on_drop(|paths: &ExternalPaths, window, cx| { + if let Some(table) = cx + .try_global::() + .and_then(|handle| handle.0.upgrade()) + { + table.update(cx, |table, cx| { + table.import_external_paths(paths.paths().to_vec(), window, cx) + }); + } + }) .flex() .flex_col() .items_center() @@ -1433,6 +1446,17 @@ impl Render for DetailsPanel { // text-undo mid-edit. div() .size_full() + .drag_over::(|style, _, _, cx| style.bg(cx.theme().secondary_hover)) + .on_drop(|paths: &ExternalPaths, window, cx| { + if let Some(table) = cx + .try_global::() + .and_then(|handle| handle.0.upgrade()) + { + table.update(cx, |table, cx| { + table.import_external_paths(paths.paths().to_vec(), window, cx) + }); + } + }) // The whole panel gives the bottom-strip crop back at once, rather than each scrolling // region padding itself: the split below sizes its panes against whatever height it is // handed, so a panel that grew 29px when the bottom dock closed re-scaled the image diff --git a/crates/workspace/src/panels/history.rs b/crates/workspace/src/panels/history.rs index 887a952d..be35227e 100644 --- a/crates/workspace/src/panels/history.rs +++ b/crates/workspace/src/panels/history.rs @@ -1202,6 +1202,7 @@ mod tests { &headers, &[1], &[vec![(from + count).to_string()]], + None, &log, ) .unwrap(); diff --git a/crates/workspace/src/views/mod.rs b/crates/workspace/src/views/mod.rs index 05d67bce..fc371d90 100644 --- a/crates/workspace/src/views/mod.rs +++ b/crates/workspace/src/views/mod.rs @@ -24,7 +24,10 @@ use gpui_component::{ tab::{Tab, TabBar}, v_flex, }; -use table::{QrateTableDelegate, Replace, Search, TableChanged, TablePanel, TableStateHandle}; +use table::{ + ImportFiles, QrateTableDelegate, RelinkMissingFiles, Replace, Search, TableChanged, TablePanel, + TableStateHandle, +}; use crate::ViewerScope; @@ -373,6 +376,20 @@ impl Render for ViewsPanel { .id("views-panel") .role(Role::Group) .aria_label("Views") + .drag_over::(|style, _, _, cx| style.bg(cx.theme().secondary_hover)) + .on_drop(cx.listener(|this, paths: &ExternalPaths, window, cx| { + this.table.update(cx, |table, cx| { + table.import_external_paths(paths.paths().to_vec(), window, cx) + }) + })) + .on_action(cx.listener(|this, _: &ImportFiles, window, cx| { + this.table + .update(cx, |table, cx| table.choose_import_paths(window, cx)) + })) + .on_action(cx.listener(|this, _: &RelinkMissingFiles, window, cx| { + this.table + .update(cx, |table, cx| table.choose_files_root(window, cx)) + })) // The find bar lives above whichever view is showing, so its keys are handled here too. .on_action(cx.listener(|this, _: &Search, window, cx| { this.table diff --git a/docs/dev/filesystem-collections-plan.md b/docs/dev/filesystem-collections-plan.md new file mode 100644 index 00000000..e6ec7682 --- /dev/null +++ b/docs/dev/filesystem-collections-plan.md @@ -0,0 +1,487 @@ +# Filesystem collections implementation plan + +## Outcome + +qrate will create and extend archival hierarchies from files and folders. A directory can become +an archival component, its children can become subordinate components, and every component remains +a normal editable row. The table will show the hierarchy inline with disclosure controls and allow +components or complete subtrees to be reordered and reparented. + +Files and folders can enter qrate through the launcher, project wizard, open table, gallery, +Details panel, Filename cells, and Project Settings. Browse buttons remain available and call the +same import pipeline as drops. + +The release name is **Filesystem Collections**. The user-facing release line is: + +> Build projects from files and folders, add material by dropping it into an open project, and +> arrange nested components without leaving the table. + +## Implementation status + +- In progress: shared filesystem inventory, relative-path preservation, and adoption by project + creation and open-project file resolution. +- Pending: schema version 4, hierarchical table behavior, import preview, drop surfaces, and + interchange contracts. + +## Archival model + +RAD and DACS describe archival material at multiple levels and require the relationship between a +component and the next higher component to remain clear. RAD commonly names fonds, collection, +series, file, and item; it also permits subdivisions where the arrangement requires them. DACS does +not prescribe one mandatory set of levels. RiC expresses broader whole-part and hierarchical +relationships rather than requiring one fixed tree vocabulary. + +qrate will follow the shared structural requirements instead of treating one standard's English +labels as its database model: + +- Every archival description is a row, including fonds, collections, series, files, and items. +- Each row has at most one immediate parent in the table hierarchy. +- A parent can have any number of ordered children. +- A row has a level-of-description term from the active profile or a project-defined term. +- The relationship is stored independently of visible metadata columns. +- Information in a parent row is not copied into its children. +- A component without a linked file is valid. +- The hierarchy reflects the current intellectual arrangement. It need not remain identical to the + disk hierarchy after import. + +The first profiles will be: + +| Profile | Initial level terms | Default folder level | Default file level | +|---|---|---|---| +| RAD | Fonds, Collection, Sous-fonds, Series, Subseries, File, Item | Series | Item | +| DACS | Collection, Record Group, Series, Subseries, File, Item | Series | Item | +| ISAD(G) | Fonds, Sub-fonds, Series, Subseries, File, Item | Series | Item | +| RiC | Record Set, Record, Record Part | Record Set | Record | +| Custom | User-defined ordered terms | User choice | User choice | + +These are editable project defaults, not validation of a single permitted depth sequence. A RAD +project can skip a level or use a local subdivision. The UI stores stable vocabulary keys and +separate display labels so a project can rename `Series` to `Series / grouping`, translate it, or +add a local level without rewriting relationships. + +## User-visible columns + +Folder import will not force `Collection`, `Parent collection`, `Relative path`, or similar columns +into the dataset. The hierarchy and source locator are project structure. + +The existing column type combines a value shape with a semantic role. `Title` and `Filename` are +already roles in practice. Extend the current model with these special types: + +- `Level of Description`: an optional visible projection of the row's internal level. +- `Parent Reference`: an optional visible projection of its immediate parent's identifier or title. + +Users can name these columns anything. During project creation they map source columns to Title, +Filename, Level of Description, and Parent Reference roles. Title and Filename remain required for +a folder-backed project. The latter two are optional because folder import can populate internal +structure without exposing implementation columns. + +When a projected structural cell is edited, qrate validates and applies the corresponding +structural operation. It must not permit a displayed Parent Reference value to disagree with the +stored hierarchy. Ambiguous title references require the user to choose a specific component; +identifiers are preferred when available. + +In a later cleanup, split `ColumnType` into value kind and semantic role. That migration is not a +prerequisite for this release and should not be bundled unless adding the two structural types makes +the current enum materially confusing in implementation. + +## Project schema version 4 + +Keep user metadata in `dataset_main`. Add a private table keyed by the stable row IDs already stored +there: + +```sql +CREATE TABLE __row_structure ( + row_id INTEGER PRIMARY KEY, + parent_id INTEGER REFERENCES dataset_main(_row_id) ON DELETE CASCADE, + level_key TEXT NOT NULL, + sibling_order INTEGER NOT NULL, + source_path TEXT, + source_kind TEXT CHECK (source_kind IN ('file', 'directory')) +); +``` + +SQLite treats `NULL` parent IDs as distinct in unique indexes, so root sibling order cannot be +enforced with a simple `(parent_id, sibling_order)` constraint. Normalize and validate sibling order +in the structure writer rather than adding a constraint that protects children but gives roots a +false appearance of protection. + +`source_path` is the normalized path relative to the project's files root when possible. It is +internal link metadata, not an archival description column. Absolute paths are allowed only for an +explicit externally linked file. Store paths with `/` separators in SQLite and convert at the OS +boundary. + +Add profile configuration to `__settings`: + +- `description_profile`: `rad`, `dacs`, `isadg`, `ric`, or `custom`. +- `description_levels`: ordered JSON records containing stable key, display label, and optional + broader default. +- `folder_level_key` and `file_level_key`: import defaults. +- `table_hierarchy_expanded`: the expanded row IDs, persisted as presentation state. + +### Migration behavior + +Opening a version 3 project remains read-only. Do not create `__row_structure` merely by opening it. +On the first hierarchy write, create the table in a transaction, set every existing row to a root +component with the project's default item level, write the requested change, then set +`PRAGMA user_version = 4`. + +New projects create the table immediately. Loading treats an absent structure row as an ungrouped +root, which keeps partially migrated or manually edited projects usable. Save dataset and structure +in one transaction so a crash cannot leave parent IDs referring to the previous dataset rewrite. +The current `save_dataset` drops and recreates `dataset_main`; it must be replaced with a transaction +that temporarily preserves or rewrites `__row_structure` around that operation. + +Validate on load and before save: + +- all referenced row IDs exist; +- a row is never its own parent; +- following parents always terminates; +- sibling order is unique and contiguous after normalization; +- stored level keys exist in the project vocabulary; +- source paths do not escape the files root unless marked external. + +Invalid relationships become Problems findings. qrate should detach a cycle or dangling component +only through an explicit accepted fix, never silently. + +## Crate changes + +### New `crates/file-ingest` crate + +This crate owns filesystem discovery and pure import planning. It has no GPUI or project-state +dependency. + +Core types: + +```rust +pub struct ScanRequest { + pub roots: Vec, + pub recursive: bool, + pub include_hidden: bool, + pub follow_directory_symlinks: bool, +} + +pub struct DiscoveredEntry { + pub root: PathBuf, + pub absolute_path: PathBuf, + pub relative_path: PathBuf, + pub kind: EntryKind, + pub parent: Option, +} + +pub struct ImportPlan { + pub components: Vec, + pub warnings: Vec, +} +``` + +The scanner will preserve relative paths, sort deterministically by normalized path, ignore existing +OS cruft, detect duplicate names, report unreadable entries, and avoid following directory symlinks +by default. A visited-directory identity guard is still required when following links is enabled. + +Move the wizard's `list_files`, matching inventory, ignored-file rules, and the recursive traversal +inside `PhotoIndex` onto this shared inventory. Filename-key matching remains in `settings` because +it is also used outside import. + +Scan work runs on GPUI's background executor in its callers. A new scan supersedes the previous +request; results carry a request generation so a late scan cannot overwrite a newer drop. + +### `settings` + +- Add schema-v4 structure types and read/write operations. +- Extend `ProjectData` with row structure aligned by stable row ID, not source index. +- Add lazy v3-to-v4 write migration. +- Save dataset rows and row structure atomically. +- Add project profile and level-vocabulary settings. +- Add `LevelOfDescription` and `ParentReference` column types. +- Add structural validators that return actionable errors without depending on the table crate. + +### `project-wizard` + +- Replace `FolderMatch.extra_files: Vec` with a complete `ImportPlan`. +- Accept dropped spreadsheet, column-config, file, and directory paths. +- Add an Arrange step for folder-backed projects. +- Let users choose a standards profile, rename its suggested levels, add local levels, and choose + the default folder and file levels. +- Map Title, Filename, optional Level of Description, and optional Parent Reference columns. +- Preview the generated tree and counts before creation. +- Pass initial row structure through `ProjectSpec`; do not reconstruct it from filenames in the + settings crate. + +### `table` + +- Replace the single-purpose row insertion API with batch insertion of complete rows and structure. +- Extend history rows to carry parent, level, sibling order, and source path. +- Add structural operations for insert subtree, delete subtree, move before/after/into, reparent, + change level, indent, outdent, and group selected rows. +- Compute visible rows from filters plus expanded ancestors. +- Render disclosure controls and indentation in the pinned row header. +- Add external drop handlers and one import-preview entry point shared by table and gallery. +- Refresh file resolution from stored source paths first, then retain filename-key fallback for old + projects and manually entered values. +- Emit one `TableChanged`, diagnostics alignment, dirty mark, and autosave per completed operation. + +### `workspace` + +- Route gallery and Details drops into the table import command. +- Make gallery order follow the visible hierarchical table order. +- Show parent components as ordinary cards when the active view includes them. +- Add breadcrumb context to Details so the selected component's ancestors are visible. + +### `app` + +- Add File menu commands for Import Files or Folders and Relink Missing Files. +- Route launcher-level external paths by type. +- Preserve relative directory layout in ZIP export. +- Add the standards profile and vocabulary editor to Project Settings. + +### Other contracts + +- `diagnostics`: use row IDs for hierarchy findings; validate cycles, dangling parents, missing + source paths, and invalid projected structural values. +- `data-exchange`: export hierarchy through format-specific mappings. CSV and Sheets can optionally + project level and parent fields with user-chosen headers. JSON-LD should express explicit + whole-part relationships. ZIP preserves directories. +- `plugin-api` and `plugin-host`: expose row ID, parent row ID, level key/label, children, and source + locator in command context and row hooks. This changes copied plugin contracts and therefore + requires the synchronized repositories described in `CLAUDE.md`. +- `ai`, table agent adapter, and `AGENTS.md`: add hierarchy fields to protocol row selection and + query results only when requested. Add revision-bound hierarchy mutation proposals only if the + bridge later gains structural draft operations; this release does not grant direct writes. + +## Table presentation + +The pinned row-number column becomes the tree gutter: + +```text +▾ 1 Collection A + ▾ 2 Photographs + 3 portrait.jpg + 4 picnic.jpg + ▸ 5 Correspondence + 8 Unarranged item +``` + +The chevron is present only when a component has children. Indentation is based on ancestor depth +and capped visually so a malformed or extremely deep tree cannot consume the table width. The +source row number remains visible and stable enough for current diagnostics references. + +Collapse state is presentation state and is not undoable. Structural changes are data changes and +are undoable. Collapsing a selected subtree keeps the parent selected and removes hidden descendants +from action targets. Expanding restores descendants without selecting them. + +Filtering uses ancestor context: + +- A matching component is visible. +- Its ancestors are visible so the match has context. +- Descendants that do not match remain hidden unless the filter itself matches the parent and the + user selects “include descendants of matching components.” +- Collapse still applies inside the filtered result. +- Search next/previous traverses only visible results. + +Sorting a column offers two modes: + +- **Within each parent** sorts siblings and preserves the hierarchy. This is the default. +- **Flat results** temporarily displays a flat sorted view and disables structural drag handles. + +Sorting does not rewrite sibling order until the user explicitly chooses “Make this order +permanent.” + +## Moving and reparenting cases + +Internal row drag is distinct from external file drop. The row header starts a component drag; file +drops remain active over cells and empty table space. + +Each row drag shows three target zones: + +- upper edge: move before the target as its sibling; +- centre: make the dragged component a child of the target; +- lower edge: move after the target as its sibling. + +The operation moves the entire subtree. Multi-selection first removes selected descendants whose +ancestor is also selected, then moves the remaining disjoint subtrees in visible order. + +| Case | Behavior | +|---|---| +| Move a leaf before or after a sibling | Change sibling order only. | +| Move a parent | Move its complete subtree. | +| Drop onto a component | Reparent as its last child. | +| Drop onto itself or a descendant | Reject because it creates a cycle. | +| Drop between different parents | Use the target sibling's parent. | +| Indent | Make it the last child of its previous visible sibling. | +| Outdent | Make it the next sibling after its current parent. | +| Move filtered rows | Allow only when parent and destination context are visible. | +| Move in flat-sort mode | Disable and explain that the hierarchy is not visible. | +| Move several selected siblings | Preserve their relative order. | +| Move selections from several parents | Preview the resulting parents before applying. | +| Delete a parent | Ask whether to delete the subtree or promote its children. | +| Duplicate a parent | Duplicate its complete subtree with fresh row IDs. | +| Cut and paste a parent | Move its complete subtree. | +| Copy and paste a parent | Duplicate its complete subtree. | +| Insert above or below | Create a sibling using the active profile's inferred level. | +| Insert child | Create the first or last child using the configured next-level default. | + +One history step stores the pre- and post-operation structure plus affected rows. Undo restores row +content, stable IDs where restoring a deletion, source paths, parents, levels, order, notes, and +resolved previews together. + +## External drop behavior + +### Launcher + +- One `.qrate` file opens the project. +- One spreadsheet starts Spreadsheet + folder and preloads it. +- Folders or ordinary files start a folder-backed project. +- A mixed drop containing `.qrate` and import material asks the user which action to take. + +### Wizard + +- Spreadsheet fields accept supported spreadsheet files. +- Folder fields accept one directory. +- The main Files area accepts multiple files and folders and creates an import plan. +- Column configuration accepts a supported config file. +- Invalid drop types leave existing valid selections unchanged. + +### Open table and gallery + +- Dropping one file on a Filename cell sets that row's link. +- Dropping several files on a row offers “add as children” and “add as siblings.” +- Dropping a folder on a row defaults to adding its generated directory component as a child. +- Dropping on empty table or gallery space appends root components. +- Dropping between rows imports at that sibling position. +- Large or recursive drops always show the import preview. +- A small file drop can apply immediately and offer Undo. + +### Files outside the project root + +The preview offers: + +1. Set the common ancestor as the new files root, with a resolution impact summary. +2. Link the files externally by absolute path. +3. Cancel. + +qrate continues to link files in place. Copying files into a managed project directory is separate +future work and must not be implied by the word “import.” + +## Import preview + +The preview is mandatory for folders, more than a small threshold of files, duplicate matches, root +changes, and any warning. It contains: + +- a collapsible tree with included/excluded controls; +- recursive scan toggle; +- hidden-file toggle; +- default folder and file levels; +- destination parent; +- title and filename mapping; +- duplicate and existing-row matches, resolved by the policy in [`import-duplicates-plan.md`](import-duplicates-plan.md); +- unreadable and skipped paths; +- final component and file counts. + +Changing an option rebuilds the pure `ImportPlan`; it does not mutate the project. Apply performs +one transaction and one undoable table operation. + +Existing row matching defaults to stored source path, then exact normalized relative path, then the +current filename-key heuristic. Ambiguous matches never merge automatically. + +## Delivery sequence + +### 1. Shared filesystem inventory + +- Add `file-ingest` to the workspace. +- Move recursive traversal and ignored-file behavior into it. +- Preserve relative paths and directory entries. +- Adapt wizard matching and `PhotoIndex` without changing visible behavior. +- Test unreadable subtrees, symlink loops, hidden files, duplicate basenames, Unicode paths, and + deterministic ordering on all supported platforms. + +### 2. Schema and model + +- Add schema-v4 structure types and lazy migration. +- Make project creation write initial hierarchy. +- Make dataset autosave atomic with structure. +- Add profile vocabulary settings and special column types. +- Update project schema tests and the migration example tools. + +### 3. Hierarchical table + +- Add hierarchy-aware view-to-source mapping. +- Render depth and disclosure controls. +- Implement collapse, expand, expand all, and collapse all. +- Implement subtree insert, delete, duplicate, indent, outdent, reorder, and reparent with undo. +- Define filter, search, selection, copy/paste, notes, diagnostics, and gallery behavior through + focused tests. + +### 4. Import planner and wizard + +- Add standards-profile selection and editable terms. +- Add folder-tree preview and column-role mapping. +- Replace flattened extra-file rows with planned components. +- Keep Browse and drop paths on the same state transitions and validators. + +### 5. External drops in the open project + +- Add GPUI `ExternalPaths` handling at launcher, wizard, table, gallery, Details, Filename cells, + and Project Settings. +- Add visual drag-over states and accessible equivalent commands. +- Add the import preview and route every surface through it. + +### 6. Interchange and contracts + +- Add hierarchy projections for CSV and Sheets. +- Add whole-part relationships to JSON-LD. +- Preserve source directories in ZIP. +- Update plugin contracts and their copied repositories. +- Update agent protocol documentation where its read surface changes. + +### 7. Release finish + +- Update user documentation and screenshots. +- Add upgrade and rollback notes for schema version 4. +- Test Windows, macOS, and Linux external drops, including network and removable drives. +- Run the full repository CI gate before the implementation PR leaves draft. + +## Acceptance scenarios + +1. Dropping a folder with two nested image folders creates parent components and item components, + preserves both duplicate basenames, and links each preview to the correct file. +2. A user can rename generated folder components and assign different RAD levels without changing + source paths. +3. A user can collapse a series, filter for a descendant, and still see enough ancestors to + understand the result. +4. Moving a series moves its descendants, survives save and reopen, and undoes in one step. +5. Reparenting onto a descendant is rejected without changing row order or dirty state. +6. Deleting a parent offers subtree deletion or child promotion, and either choice undoes fully. +7. A v3 project opens without a write, gains structure on its first hierarchy edit, and reopens in + an older release with its dataset intact even though the older release ignores hierarchy. +8. CSV and Sheets export use user-selected structural field names; a project that does not request + those fields gains no extra columns. +9. ZIP export preserves nested paths and does not overwrite files with duplicate basenames. +10. Keyboard users can perform every structural operation available through dragging. + +## Decisions to keep explicit during implementation + +- Parent rows are archival components and remain available to search, diagnostics, plugins, + exports, Sheets, gallery, notes, and agent queries. +- Disk layout seeds arrangement but does not own it after import. +- Hierarchy is stored by stable row ID and never inferred from visual adjacency. +- Source paths are private project structure unless a user maps them into a visible column. +- Standards profiles provide vocabulary and mapping defaults; they do not forbid local practice. +- Dragging is an interaction layer over commands. Browse, menus, and keyboard actions invoke the + same operations. +- No scan mutates project data before the import plan is accepted. + +## Primary standards references + +- Canadian Council of Archives, [*Rules for Archival Description*, Chapter + 1](https://www.archivescanada.ca/wp-content/uploads/2022/08/RAD_Chapter01_July2008.pdf), + especially rules 0.27 through 0.29 and 1.0D. +- Society of American Archivists, [*Describing Archives: A Content + Standard*](https://files.archivists.org/pubs/DACS2E-2013_v0315.pdf), Statement of Principles and + Chapter 1, Levels of Description. +- International Council on Archives, [*ISAD(G): General International Standard Archival + Description*, second + edition](https://www.ica.org/app/uploads/2023/12/CBPS_2000_Guidelines_ISADG_Second-edition_EN.pdf), + multilevel description rules. +- International Council on Archives, [*Records in Contexts Conceptual Model* + 1.0](https://www.ica.org/app/uploads/2023/12/RiC-CM-1.0.pdf) and [RiC-O + 1.1](https://www.ica.org/standards/RiC/RiC-O_1-1.html), whole-part and hierarchical relations. diff --git a/docs/dev/import-duplicates-plan.md b/docs/dev/import-duplicates-plan.md new file mode 100644 index 00000000..54abc662 --- /dev/null +++ b/docs/dev/import-duplicates-plan.md @@ -0,0 +1,169 @@ +# Import duplicates plan (ASNT-77, #87) + +Folds [#87](https://github.com/devnull03/qrate/issues/87) into the Filesystem Collections branch +(draft PR #134). That branch turned every import into a component plan, so the duplicate policy +belongs to the planner rather than the old flat `match_folder` extra-files list. + +## Why it lands here + +Digitization arrives in batches. The second batch is usually dropped onto a project that already +holds the first, often as the same parent folder with new files in it. On this branch today: + +- **Open table:** `append_components` never looks at existing rows. Dropping a folder that is + already imported creates a second copy of every component. +- **Wizard, two rows name one file:** `append_folder_components` collects claims into a + `HashMap`, so the last row wins and the other row silently stays unlinked. +- **Wizard, one row names several files** (the same basename in two subfolders): the row claims + nothing, and each file gets a new row. The row and the files are now duplicates of each other. +- **Wizard, preview and creation disagree:** `match_folder` indexes files with + `settings::filenames::keys`, which includes id prefixes. Creation indexes them with + `lookup_keys`, which does not. A multi-part item (`2020_04_001` → `2020_04_001_001.jpg`) is + counted as matched in the Files step, then not linked at creation, so its files arrive as new + rows beside the row that names them. + +## What counts as a duplicate + +An incoming file is checked against existing components in this order. The first rule that decides +wins: + +1. **Same stored source.** An existing component's `source_path`, resolved against the files root, + is the same file as the incoming absolute path. This is exact and is the only rule that applies + to directories. +2. **Same filename key, exactly one candidate.** The incoming file's `filenames::keys` hit exactly + one existing row's Filename cell (`lookup_keys`). This is the rule that finds a spreadsheet row + for its file. +3. **Same filename key, several candidates.** This is **ambiguous**. It is reported and never + merged automatically, whatever the policy. + +Inside one import, the same absolute path reached twice (a folder plus a file inside it dropped +together) collapses to one component without asking. That is a single file, not a choice. + +Rows with identical *metadata* are not duplicates. Two rows can describe the same photograph on +purpose, and qrate's Problems panel is where that gets flagged, not the importer. + +## The policy + +```rust +pub enum DuplicatePolicy { + /// Leave the existing component alone and drop the incoming one. The default: re-importing a + /// batch folder is the common case and must be a no-op for what is already there. + Skip, + /// Keep the existing row and its typed metadata; re-link it to the incoming file and move it + /// to the imported position in the hierarchy. + Update, + /// Import it as a new component anyway. + AddAsNew, +} +``` + +**"Overwrite" deliberately means `Update`, not replacement.** qrate links files in place and copies +nothing, so the only thing an import can bring that the row lacks is *where the file is* and *where +it sits in the arrangement*. Blanking a catalogued row because its file was dropped again would +destroy the archivist's work for no gain, so no policy clears cells. + +Ambiguous matches ignore the policy. They are listed in the preview, and each one becomes a new +component unless the archivist picks a target row there. + +The chosen policy is saved per project in `__settings` as `import_duplicate_policy`, so the next +batch starts with the same answer. + +## Design + +### `file-ingest`: one pure resolver + +A new `duplicates` module. It knows nothing about the project, so the caller hands it plain facts: + +```rust +pub struct ExistingComponent { + pub key: u64, // the caller's RowId + pub absolute_source: Option, + pub filename_keys: Vec, // lookup_keys of the Filename cell +} + +pub enum Resolution { + Create, + Skip { existing: u64 }, + Update { existing: u64 }, + Ambiguous { candidates: Vec }, +} + +pub struct ResolvedPlan { + pub plan: ImportPlan, // within-import duplicates already collapsed + pub resolutions: Vec, // parallel to plan.components +} + +pub fn resolve( + plan: ImportPlan, + existing: &[ExistingComponent], + file_keys: impl Fn(&str) -> Vec, + policy: DuplicatePolicy, +) -> ResolvedPlan; +``` + +`file_keys` is passed in as `settings::filenames::keys` so both callers use one key rule, which +fixes the preview/creation disagreement. A skipped directory still anchors its children: an incoming +file under a skipped folder is parented to the *existing* folder row, so a second batch lands inside +the first batch's series instead of in a new copy of it. + +### Wizard + +- `match_folder` runs `resolve` against the spreadsheet rows, which are the "existing components" + here, and `FolderMatch` gains `duplicate_rows`, `ambiguous_files`, and the resolved plan. + `extra_files` goes away. +- The Files step shows the counts and, only when there is at least one duplicate or ambiguous + match, a three-way choice for the policy. +- `append_folder_components` consumes the `ResolvedPlan` instead of re-scanning and re-matching. For + a new project, `Update` and `Skip` differ only for rows that claim the same file: `Update` links + the first row by sheet order and reports the rest, and `Skip` links none of them. + +### Open table + +- `import_external_paths` builds `ExistingComponent`s from the live delegate (structure + `source_path` plus Filename cells) and resolves on the background executor. +- The prompt names the duplicate counts and offers `Skip duplicates`, `Update existing`, + `Add all as new`, and `Cancel`. A drop with no duplicates keeps today's two buttons. +- `append_components` takes the `ResolvedPlan`. `Create` rows are appended. `Update` becomes a + Filename cell change plus a structure change on the existing row. Everything goes in **one** + history step: `Step::RowsAdded` gains a `cells` field carrying those before/after cell values, + since it already carries before/after structure. + +## Status + +Steps 1-3 are implemented on `feat/filesystem-collections-plan`. Two things differ from the design +above, both deliberately: + +- **`Update` re-links, it does not re-arrange.** The existing row keeps its place in the hierarchy; + only its Filename cell and stored source path follow the file. Moving a catalogued component + because its file was dropped again would undo arrangement the archivist did by hand. +- **The wizard's control appears only when the folder holds a file that several rows name**, which + is the one ambiguity project creation cannot settle on its own. A file matching exactly one row + always links that row, since that is what a folder-backed project is for. + +## Delivery + +Each step is one commit on `feat/filesystem-collections-plan`. + +1. **Resolver.** `file_ingest::duplicates` with unit tests for every rule and policy: exact source, + single key, ambiguous, within-import collapse, and a skipped parent adopting new children. +2. **Wizard.** Use `keys` on the file side, route `match_folder` and creation through `resolve`, + add the policy control and persisted setting. Tests: two rows naming one file under each policy, + one row naming two same-named files, and a multi-part item matched in preview and at creation. +3. **Open table.** Resolve before the prompt, apply `Update` in the same undo step. A gpui test + drops the same folder twice under `Skip` (no new rows, undo restores) and under `Update` (the + moved file is re-linked, typed cells untouched). +4. **Docs and trackers.** Add a Duplicates subsection to the Import preview section of + `filesystem-collections-plan.md`. Add a memory-log entry to the ASNT-77 Notion page, link #87 + from PR #134 (`Closes #87 (Notion ID: 3b821d32-b13b-818d-a1b6-c542b16fe2d7)`), and mirror the + status in Linear. + +## Definition of done + +From #87, adapted to this branch: `./scripts/ci.sh` is green, and importing a folder/spreadsheet +combination with an intentional duplicate produces the chosen behaviour, covered by tests next to +the matching code in both the wizard and the open table. + +## Out of scope + +- Content hashing to spot the same file under a different name or path. That is expensive on + network drives, and the source-path and key rules cover the batch workflow #87 describes. +- Duplicate *metadata* detection between rows, which belongs to diagnostics.