From 05d9b2c29a11a337924e1e64ee84227d438630fe Mon Sep 17 00:00:00 2001 From: devnull03 <56480041+devnull03@users.noreply.github.com> Date: Sun, 13 Sep 2026 16:17:03 -0700 Subject: [PATCH 01/12] feat(table): search text inside linked PDFs Table search gains a linked-files toggle, on by default. Each row's linked document is read once in the background, cached on disk beside the thumbnails, and matched with the same case/word/regex rules as cells. A hit lands on the cell that names the file. --- crates/preview/src/lib.rs | 26 ++++++++ crates/preview/src/pdf.rs | 39 +++++++++++ crates/table/src/delegate.rs | 123 +++++++++++++++++++++++++++++++++-- crates/table/src/panel.rs | 54 ++++++++++++++- 4 files changed, 235 insertions(+), 7 deletions(-) diff --git a/crates/preview/src/lib.rs b/crates/preview/src/lib.rs index bd31a22b..407e662c 100644 --- a/crates/preview/src/lib.rs +++ b/crates/preview/src/lib.rs @@ -346,6 +346,32 @@ pub fn has_text_layer(path: &Path) -> Option { pdf::has_text_layer(path) } +/// The whole text of a document, for searching across a collection rather than inside one file. +/// Cached on disk beside the thumbnails, so a project's documents are read once, not every launch. +/// `None` for anything that is not a document, and where PDFium is not installed. +pub fn document_text(path: &Path) -> Option { + if !has_text(path) { + return None; + } + let entry = cache::key(path, FULL, 0).and_then(|key| Some(cache::dir()?.join(key + ".txt"))); + if let Some(text) = entry + .as_ref() + .and_then(|entry| std::fs::read_to_string(entry).ok()) + { + return Some(text); + } + let text = pdf::text(path)?; + if let Some(entry) = entry + && let Err(err) = std::fs::write(&entry, &text) + { + log::warn!( + "could not cache the text of {}, it will be read again next launch: {err}", + path.display() + ); + } + Some(text) +} + /// Every hit for `needle` in this file, in document order. Empty for a blank query, for anything /// that is not a document, and where PDFium is not installed. pub fn search(path: &Path, needle: &str) -> Vec { diff --git a/crates/preview/src/pdf.rs b/crates/preview/src/pdf.rs index 59e7a234..354c2415 100644 --- a/crates/preview/src/pdf.rs +++ b/crates/preview/src/pdf.rs @@ -94,6 +94,23 @@ pub fn page_count(path: &Path) -> Option { usize::try_from(document.pages().len()).ok() } +/// The text layer of every page, one page per line break. Empty for a scan that was never OCR'd. +/// +/// ponytail: holds the PDFium lock for the whole document, so a 500-page PDF stalls gallery +/// thumbnails until it is read. Per-page locking if that shows up in a real collection. +pub fn text(path: &Path) -> Option { + let pdfium = locked()?; + let document = pdfium.load_pdf_from_file(path, None).ok()?; + Some( + document + .pages() + .iter() + .filter_map(|page| page.text().ok().map(|text| text.all())) + .collect::>() + .join("\n"), + ) +} + /// Whether the document carries a text layer at all. /// /// A scan that was never OCR'd has none, and "no matches" is a misleading answer to every search @@ -404,6 +421,28 @@ trailer<>"; let _ = std::fs::remove_file(&typed); } + /// Collection search reads every page, and reads it again from the cache on the next launch. + #[test] + fn document_text_covers_every_page_and_survives_the_cache() { + let path = document_with_text("whole"); + let Some(text) = crate::document_text(&path) else { + eprintln!("skipping the document text check: PDFium is not installed"); + return; + }; + assert!(text.contains("Aderman family") && text.contains("Kitsilano beach")); + assert_eq!( + crate::document_text(&path), + Some(text), + "the cached copy reads back" + ); + assert_eq!( + crate::document_text(std::path::Path::new("photo.jpg")), + None + ); + + let _ = std::fs::remove_file(&path); + } + /// Paging has to actually select a different page, not just re-render the first one. The two /// pages are deliberately opposite orientations, which is observable in the result without /// needing any page content to draw. diff --git a/crates/table/src/delegate.rs b/crates/table/src/delegate.rs index 6e7efc4d..943f77e4 100644 --- a/crates/table/src/delegate.rs +++ b/crates/table/src/delegate.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeSet, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::ops::RangeInclusive; use std::path::{Path, PathBuf}; @@ -73,6 +73,9 @@ pub struct QrateTableDelegate { pub(crate) note_editor: Entity, /// Each row's resolved image path, parallel to `rows`. `None` until `TablePanel` resolves it. image_paths: Vec>, + /// The text inside linked documents, by path. Filled by `TablePanel` the first time a search + /// includes linked files; a file with no text layer holds an empty string so it is not re-read. + document_text: HashMap, /// View→source row mapping: `visible_rows[view] == source`. The library only ever sees this /// narrowed set, so filtering composes with the virtualized render for free. visible_rows: Vec, @@ -112,6 +115,7 @@ impl QrateTableDelegate { note_edit: None, note_editor, image_paths: Vec::new(), + document_text: HashMap::new(), visible_rows: Vec::new(), filters: Vec::new(), filters_enabled: Vec::new(), @@ -321,13 +325,44 @@ impl QrateTableDelegate { /// Cells in the *visible* set matching `needle` under `opts`, as `(view_row, data_col)` in view /// order — ready for `set_selected_cell`/`scroll_to_row` after the pinned `+1` on the column. pub(crate) fn search_matches(&self, needle: &str, opts: SearchOpts) -> Vec<(usize, usize)> { - find_matches( + let mut hits = find_matches( &self.rows, &self.visible_rows, self.columns.len(), needle, opts, - ) + ); + if opts.files { + hits.extend(find_file_matches( + &self.rows, + &self.visible_rows, + &self.image_paths, + &self.document_text, + needle, + opts, + )); + hits.sort_unstable(); + hits.dedup(); + } + hits + } + + /// Linked documents whose text has not been read yet, each once. + pub(crate) fn unread_documents(&self) -> Vec { + let mut unread: Vec = self + .image_paths + .iter() + .flatten() + .filter(|path| preview::has_text(path) && !self.document_text.contains_key(*path)) + .cloned() + .collect(); + unread.sort_unstable(); + unread.dedup(); + unread + } + + pub(crate) fn add_document_text(&mut self, texts: Vec<(PathBuf, String)>) { + self.document_text.extend(texts); } /// The cell writes replacing `needle` with `replacement` produces, in *source* coordinates and @@ -1070,6 +1105,8 @@ pub(crate) struct SearchOpts { pub case: bool, pub word: bool, pub regex: bool, + /// Also match the text inside each row's linked document. + pub files: bool, } /// Compile the query into a matcher honoring the toggles. `None` means "match nothing": a blank @@ -1121,6 +1158,41 @@ fn find_matches( hits } +/// Rows whose linked document contains `needle`, addressed at the cell that names the file so +/// stepping to the hit lands where the archivist would look for it. +fn find_file_matches( + rows: &[Vec], + visible: &[usize], + image_paths: &[Option], + document_text: &HashMap, + needle: &str, + opts: SearchOpts, +) -> Vec<(usize, usize)> { + let Some(re) = compile_search(needle, opts) else { + return Vec::new(); + }; + visible + .iter() + .enumerate() + .filter_map(|(view, &source)| { + let path = image_paths.get(source)?.as_ref()?; + if !re.is_match(document_text.get(path)?) { + return None; + } + let names = path + .file_name() + .map(|name| settings::filenames::keys(&name.to_string_lossy())) + .unwrap_or_default(); + let col = rows.get(source)?.iter().position(|cell| { + settings::filenames::lookup_keys(cell) + .iter() + .any(|key| names.contains(key)) + }); + Some((view, col.unwrap_or(0))) + }) + .collect() +} + /// Rewrite matches in place: `Regex::replace_all` substitutes only the matched spans, so the rest /// of the cell's text survives and a cell with two occurrences gets both. Literal mode wraps the /// replacement in `NoExpand` so a `$` the user typed stays a `$`; regex mode leaves `$1` capture @@ -1379,6 +1451,41 @@ mod tests { assert_eq!(find_matches(&data, &visible, 2, "ap", opts), vec![(0, 0)]); } + /// A hit inside a linked PDF lands on the cell naming the file; a row whose document lacks the + /// query, or that has no document at all, is not reported. + #[test] + fn find_reaches_into_linked_documents() { + let data = rows(&[ + &["Letter to council", "1920_letter.pdf"], + &["Portrait", "1921_portrait.jpg"], + &["Minutes", "1922_minutes.pdf"], + ]); + let paths = vec![ + Some(PathBuf::from("files/1920_letter.pdf")), + Some(PathBuf::from("files/1921_portrait.jpg")), + Some(PathBuf::from("files/1922_minutes.pdf")), + ]; + let text = HashMap::from([ + ( + PathBuf::from("files/1920_letter.pdf"), + "the sawmill on Kitsilano beach".to_string(), + ), + ( + PathBuf::from("files/1922_minutes.pdf"), + "motion carried".to_string(), + ), + ]); + let opts = SearchOpts { + files: true, + ..SearchOpts::default() + }; + assert_eq!( + find_file_matches(&data, &[0, 1, 2], &paths, &text, "SAWMILL", opts), + vec![(0, 1)] + ); + assert!(find_file_matches(&data, &[1, 2], &paths, &text, "sawmill", opts).is_empty()); + } + #[test] fn blank_query_matches_nothing() { let data = rows(&[&["a"]]); @@ -1389,7 +1496,12 @@ mod tests { fn search_toggles_case_word_and_regex() { let data = rows(&[&["Apple pie"], &["pineapple"]]); let all = &[0, 1][..]; - let opt = |case, word, regex| SearchOpts { case, word, regex }; + let opt = |case, word, regex| SearchOpts { + case, + word, + regex, + files: false, + }; // Match case: "apple" no longer hits the capitalized "Apple pie". assert_eq!( @@ -1463,9 +1575,8 @@ mod tests { fn replace_expands_captures_only_in_regex_mode() { let data = rows(&[&["Smith, Jane"]]); let regex = SearchOpts { - case: false, - word: false, regex: true, + ..SearchOpts::default() }; assert_eq!( replace_edits(&data, &[0], 1, r"(\w+), (\w+)", "$2 $1", regex, None), diff --git a/crates/table/src/panel.rs b/crates/table/src/panel.rs index fc13db34..a78c7e98 100644 --- a/crates/table/src/panel.rs +++ b/crates/table/src/panel.rs @@ -93,6 +93,8 @@ pub struct TablePanel { search_opts: SearchOpts, search_error: bool, _search_sub: Subscription, + /// Linked documents being read for a search that includes them. `Some` while it runs. + reading_documents: Option>, replace_input: Entity, replace_open: bool, _replace_sub: Subscription, @@ -386,9 +388,13 @@ impl TablePanel { search_open: false, search_matches: Vec::new(), search_ix: 0, - search_opts: SearchOpts::default(), + search_opts: SearchOpts { + files: true, + ..SearchOpts::default() + }, search_error: false, _search_sub, + reading_documents: None, replace_input, replace_open: false, _replace_sub, @@ -441,9 +447,45 @@ impl TablePanel { .search_matches(&needle, self.search_opts); self.search_ix = 0; self.select_current_match(cx); + self.read_documents(cx); cx.notify(); } + /// Read the text of every linked document the search has not seen yet, off the UI thread, then + /// search again so their hits join the results. + fn read_documents(&mut self, cx: &mut Context) { + if !self.search_opts.files || self.reading_documents.is_some() { + return; + } + let unread = self.state.read(cx).delegate().unread_documents(); + if unread.is_empty() { + return; + } + self.reading_documents = Some(cx.spawn(async move |this, cx| { + let texts = cx + .background_executor() + .spawn(async move { + unread + .into_iter() + .map(|path| { + let text = preview::document_text(&path).unwrap_or_default(); + (path, text) + }) + .collect() + }) + .await; + this.update(cx, |this, cx| { + this.reading_documents = None; + this.state + .update(cx, |state, _| state.delegate_mut().add_document_text(texts)); + if this.search_open { + this.refresh_search(cx); + } + }) + .ok(); + })); + } + fn toggle_opt(&mut self, pick: fn(&mut SearchOpts) -> &mut bool, cx: &mut Context) { let flag = pick(&mut self.search_opts); *flag = !*flag; @@ -890,6 +932,8 @@ impl Render for TablePanel { )) } else if query.trim().is_empty() { SharedString::default() + } else if self.reading_documents.is_some() { + SharedString::from("Reading files…") } else { SharedString::from("No results") }; @@ -1066,6 +1110,14 @@ impl Render for TablePanel { "Use regular expression", opts.regex, |o| &mut o.regex, + )) + .child(toggle( + "search-files", + Some(IconName::BookOpen), + "", + "Include text inside linked files", + opts.files, + |o| &mut o.files, )), ) .child( From baffa619dfa7c9557e6282f1f122fce2db6a18fc Mon Sep 17 00:00:00 2001 From: devnull03 <56480041+devnull03@users.noreply.github.com> Date: Sun, 13 Sep 2026 16:47:40 -0700 Subject: [PATCH 02/12] feat(table): search images by description and find similar items Adds the visual-search crate: CLIP ViT-B/32 on the CPU through candle, a hand-rolled CLIP tokenizer, a checksum-verified download of the pinned weights, and an index of thumbnail embeddings kept on disk. The table search bar gains a Visual toggle that ranks rows by how well their linked file matches the query, offers the model download when it is missing, and indexes linked files in the background. The row and gallery context menu gains Find similar items. --- CLAUDE.md | 1 + Cargo.lock | 361 +++++++++++++++++++++++++- Cargo.toml | 1 + crates/preview/src/lib.rs | 4 +- crates/table/Cargo.toml | 1 + crates/table/src/delegate.rs | 93 ++++++- crates/table/src/lib.rs | 1 + crates/table/src/note.rs | 15 +- crates/table/src/panel.rs | 133 +++++++++- crates/table/src/visual.rs | 306 ++++++++++++++++++++++ crates/visual-search/Cargo.toml | 21 ++ crates/visual-search/src/index.rs | 189 ++++++++++++++ crates/visual-search/src/lib.rs | 206 +++++++++++++++ crates/visual-search/src/tokenizer.rs | 135 ++++++++++ 14 files changed, 1443 insertions(+), 24 deletions(-) create mode 100644 crates/table/src/visual.rs create mode 100644 crates/visual-search/Cargo.toml create mode 100644 crates/visual-search/src/index.rs create mode 100644 crates/visual-search/src/lib.rs create mode 100644 crates/visual-search/src/tokenizer.rs diff --git a/CLAUDE.md b/CLAUDE.md index 5cfbba66..0d4d7b55 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -83,6 +83,7 @@ problem — read the `--check` diff and apply it by hand rather than fighting th | `plugin-host` | the Luau runtime that loads and runs plugins | | `plugin-api` | the types a plugin sees — see the three-repo rule below | | `preview` | turns a linked file into pixels — the format ladder and the thumbnail cache. Native decoders (PDF, video, RAW) belong here so they never reach `table` | +| `visual-search` | CLIP on the CPU through candle (pure Rust), its pinned weights download, and the on-disk image index. No gpui — `table::visual` owns the job, the search bar, and "Find similar items". Weights live in `/models`, never in the install | | `ai` | two halves: `agent.rs` is the *shipped* external-agent contract (see `AGENTS.md`); the rest is traits + Cohere/mock providers for planned AI review/embedding, deliberately unfinished | ## Project Status Tracking diff --git a/Cargo.lock b/Cargo.lock index aeb059e7..2fe53dc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -981,6 +981,64 @@ dependencies = [ "wayland-client", ] +[[package]] +name = "candle-core" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c15b675b80d994b2eadb20a4bbe434eabeb454eac3ee5e2b4cf6f147ee9be091" +dependencies = [ + "byteorder", + "float8", + "gemm", + "half", + "libm", + "memmap2", + "num-traits", + "num_cpus", + "rand 0.9.2", + "rand_distr", + "rayon", + "safetensors", + "thiserror 2.0.18", + "yoke", + "zip 7.2.0", +] + +[[package]] +name = "candle-nn" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045fa9e7aef8567d209a27d56b692f60b96f4d0569f4c3011f8ca6715c65e03" +dependencies = [ + "candle-core", + "half", + "libc", + "num-traits", + "rayon", + "safetensors", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "candle-transformers" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b538ec4aa807c416a2ddd3621044888f188827862e2a6fcacba4738e89795d01" +dependencies = [ + "byteorder", + "candle-core", + "candle-nn", + "fancy-regex 0.17.0", + "num-traits", + "rand 0.9.2", + "rayon", + "serde", + "serde_json", + "serde_plain", + "tracing", +] + [[package]] name = "caseless" version = "0.2.2" @@ -1942,6 +2000,22 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "dyn-stack" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c4713e43e2886ba72b8271aa66c93d722116acf7a75555cce11dcde84388fe8" +dependencies = [ + "bytemuck", + "dyn-stack-macros", +] + +[[package]] +name = "dyn-stack-macros" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d926b4d407d372f141f93bb444696142c29d32962ccbd3531117cf3aa0bfa9" + [[package]] name = "ed25519" version = "2.2.3" @@ -2011,6 +2085,18 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099" +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "enum-iterator" version = "2.3.0" @@ -2194,6 +2280,17 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "fancy-regex" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" +dependencies = [ + "bit-set 0.8.0", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fast-float2" version = "0.2.4" @@ -2301,6 +2398,18 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" +[[package]] +name = "float8" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719a903cc23e4a89e87962c2a80fdb45cdaad0983a89bd150bb57b4c8571a7d5" +dependencies = [ + "half", + "num-traits", + "rand 0.9.2", + "rand_distr", +] + [[package]] name = "float_next_after" version = "1.0.0" @@ -2582,6 +2691,125 @@ dependencies = [ "slab", ] +[[package]] +name = "gemm" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa0673db364b12263d103b68337a68fbecc541d6f6b61ba72fe438654709eacb" +dependencies = [ + "dyn-stack", + "gemm-c32", + "gemm-c64", + "gemm-common", + "gemm-f16", + "gemm-f32", + "gemm-f64", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-c32" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "086936dbdcb99e37aad81d320f98f670e53c1e55a98bee70573e83f95beb128c" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-c64" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20c8aeeeec425959bda4d9827664029ba1501a90a0d1e6228e48bef741db3a3f" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-common" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88027625910cc9b1085aaaa1c4bc46bb3a36aad323452b33c25b5e4e7c8e2a3e" +dependencies = [ + "bytemuck", + "dyn-stack", + "half", + "libm", + "num-complex", + "num-traits", + "once_cell", + "paste", + "pulp", + "raw-cpuid", + "rayon", + "seq-macro", + "sysctl", +] + +[[package]] +name = "gemm-f16" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3df7a55202e6cd6739d82ae3399c8e0c7e1402859b30e4cb780e61525d9486e" +dependencies = [ + "dyn-stack", + "gemm-common", + "gemm-f32", + "half", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "rayon", + "seq-macro", +] + +[[package]] +name = "gemm-f32" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e0b8c9da1fbec6e3e3ab2ce6bc259ef18eb5f6f0d3e4edf54b75f9fd41a81c" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-f64" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "056131e8f2a521bfab322f804ccd652520c79700d81209e9d9275bbdecaadc6a" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -3415,9 +3643,12 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ + "bytemuck", "cfg-if", "crunchy", "num-traits", + "rand 0.9.2", + "rand_distr", "zerocopy", ] @@ -3469,6 +3700,8 @@ dependencies = [ "allocator-api2", "equivalent", "foldhash 0.2.0", + "serde", + "serde_core", ] [[package]] @@ -4576,6 +4809,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" dependencies = [ "libc", + "stable_deref_trait", ] [[package]] @@ -4965,6 +5199,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ + "bytemuck", "num-traits", ] @@ -6194,6 +6429,29 @@ dependencies = [ "syn", ] +[[package]] +name = "pulp" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046aa45b989642ec2e4717c8e72d677b13edd831a4d3b6cf37d9a3e54912496a" +dependencies = [ + "bytemuck", + "cfg-if", + "libm", + "num-complex", + "paste", + "pulp-wasm-simd-flag", + "raw-cpuid", + "reborrow", + "version_check", +] + +[[package]] +name = "pulp-wasm-simd-flag" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d8f70e07b9c3962945a74e59ca1c511bba65b6419468acc217c457d93f3c740" + [[package]] name = "pxfm" version = "0.1.28" @@ -6404,6 +6662,16 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.2", +] + [[package]] name = "rand_xorshift" version = "0.4.0" @@ -6475,6 +6743,15 @@ dependencies = [ "rgb", ] +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.11.0", +] + [[package]] name = "raw-window-handle" version = "0.6.2" @@ -6534,6 +6811,12 @@ dependencies = [ "font-types 0.11.3", ] +[[package]] +name = "reborrow" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" + [[package]] name = "redox_syscall" version = "0.5.18" @@ -7012,6 +7295,17 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safetensors" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "675656c1eabb620b921efea4f9199f97fc86e36dd6ffd1fbbe48d0f59a4987f5" +dependencies = [ + "hashbrown 0.16.1", + "serde", + "serde_json", +] + [[package]] name = "same-file" version = "1.0.6" @@ -7168,6 +7462,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + [[package]] name = "serde" version = "1.0.228" @@ -7252,6 +7552,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_plain" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50" +dependencies = [ + "serde", +] + [[package]] name = "serde_repr" version = "0.1.20" @@ -7989,7 +8298,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "656b45c05d95a5704399aeef6bd0ddec7b2b3531b7c9e900abbf7c4d2190c925" dependencies = [ "bincode", - "fancy-regex", + "fancy-regex 0.16.2", "flate2", "fnv", "once_cell", @@ -8009,6 +8318,20 @@ dependencies = [ "libc", ] +[[package]] +name = "sysctl" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" +dependencies = [ + "bitflags 2.11.0", + "byteorder", + "enum-as-inner", + "libc", + "thiserror 1.0.69", + "walkdir", +] + [[package]] name = "sysinfo" version = "0.31.4" @@ -8089,6 +8412,7 @@ dependencies = [ "serde_json", "settings", "sha2", + "visual-search", ] [[package]] @@ -8643,6 +8967,12 @@ dependencies = [ "core_maths", ] +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typeid" version = "1.0.3" @@ -8965,6 +9295,23 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "visual-search" +version = "0.4.0-beta.2" +dependencies = [ + "anyhow", + "candle-core", + "candle-nn", + "candle-transformers", + "dirs 6.0.0", + "image", + "log", + "regex", + "reqwest", + "serde_json", + "sha2", +] + [[package]] name = "vswhom" version = "0.1.0" @@ -10911,6 +11258,18 @@ dependencies = [ "zopfli", ] +[[package]] +name = "zip" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0" +dependencies = [ + "crc32fast", + "indexmap", + "memchr", + "typed-path", +] + [[package]] name = "zlib-rs" version = "0.6.7" diff --git a/Cargo.toml b/Cargo.toml index b3f3016e..4220c536 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ members = [ "crates/spellcheck", "crates/table", "crates/updater", + "crates/visual-search", "crates/window-wrapper", "crates/workspace", ] diff --git a/crates/preview/src/lib.rs b/crates/preview/src/lib.rs index 407e662c..984fb6b8 100644 --- a/crates/preview/src/lib.rs +++ b/crates/preview/src/lib.rs @@ -405,7 +405,9 @@ fn render(path: &Path, max_edge: u32, page: usize) -> Option> { Some(Arc::new(RenderImage::new([Frame::new(bgra)]))) } -fn thumbnail_pixels(path: &Path, max_edge: u32, page: usize) -> Option { +/// `path` decoded down the ladder and shrunk to fit `max_edge`, from the disk cache when it has been +/// drawn before. What a card shows, as pixels rather than something only gpui can draw. +pub fn thumbnail_pixels(path: &Path, max_edge: u32, page: usize) -> Option { // A full-size rendering is not cached: it would be a second copy of the original file on disk, // and it is wanted once, while someone is looking at it. let key = (max_edge != FULL) diff --git a/crates/table/Cargo.toml b/crates/table/Cargo.toml index 5896d37e..8360e352 100644 --- a/crates/table/Cargo.toml +++ b/crates/table/Cargo.toml @@ -17,6 +17,7 @@ plugin-api = { path = "../plugin-api" } settings = { path = "../settings" } plugin-host = { path = "../plugin-host" } preview = { path = "../preview" } +visual-search = { path = "../visual-search" } base64 = "0.22" sha2 = "0.10" diff --git a/crates/table/src/delegate.rs b/crates/table/src/delegate.rs index 943f77e4..e7fe4ded 100644 --- a/crates/table/src/delegate.rs +++ b/crates/table/src/delegate.rs @@ -347,6 +347,39 @@ impl QrateTableDelegate { hits } + /// The visible rows with a linked file that `score` can rate, best first and at most `limit` of + /// them, addressed like search hits. + pub(crate) fn ranked_matches( + &self, + score: impl Fn(&Path) -> Option, + limit: usize, + ) -> Vec<(usize, usize)> { + let mut scored: Vec<(f32, usize, usize)> = self + .visible_rows + .iter() + .enumerate() + .filter_map(|(view, &source)| { + let path = self.image_paths.get(source)?.as_ref()?; + let col = file_column(self.rows.get(source)?, path); + Some((score(path)?, view, col)) + }) + .collect(); + scored.sort_by(|a, b| b.0.total_cmp(&a.0)); + scored + .into_iter() + .take(limit) + .map(|(_, view, col)| (view, col)) + .collect() + } + + /// Every distinct file the rows link to. + pub(crate) fn linked_files(&self) -> Vec { + let mut files: Vec = self.image_paths.iter().flatten().cloned().collect(); + files.sort_unstable(); + files.dedup(); + files + } + /// Linked documents whose text has not been read yet, each once. pub(crate) fn unread_documents(&self) -> Vec { let mut unread: Vec = self @@ -1107,6 +1140,8 @@ pub(crate) struct SearchOpts { pub regex: bool, /// Also match the text inside each row's linked document. pub files: bool, + /// Rank rows by how well their linked file matches a description instead of matching text. + pub visual: bool, } /// Compile the query into a matcher honoring the toggles. `None` means "match nothing": a blank @@ -1179,20 +1214,27 @@ fn find_file_matches( if !re.is_match(document_text.get(path)?) { return None; } - let names = path - .file_name() - .map(|name| settings::filenames::keys(&name.to_string_lossy())) - .unwrap_or_default(); - let col = rows.get(source)?.iter().position(|cell| { - settings::filenames::lookup_keys(cell) - .iter() - .any(|key| names.contains(key)) - }); - Some((view, col.unwrap_or(0))) + Some((view, file_column(rows.get(source)?, path))) }) .collect() } +/// The column of the cell in `row` that names `path`, so a hit about the file lands where the +/// archivist would look for it. The first column when no cell names it directly. +fn file_column(row: &[SharedString], path: &Path) -> usize { + let names = path + .file_name() + .map(|name| settings::filenames::keys(&name.to_string_lossy())) + .unwrap_or_default(); + row.iter() + .position(|cell| { + settings::filenames::lookup_keys(cell) + .iter() + .any(|key| names.contains(key)) + }) + .unwrap_or(0) +} + /// Rewrite matches in place: `Regex::replace_all` substitutes only the matched spans, so the rest /// of the cell's text survives and a cell with two occurrences gets both. Literal mode wraps the /// replacement in `NoExpand` so a `$` the user typed stays a `$`; regex mode leaves `$1` capture @@ -1500,7 +1542,7 @@ mod tests { case, word, regex, - files: false, + ..SearchOpts::default() }; // Match case: "apple" no longer hits the capitalized "Apple pie". @@ -1848,6 +1890,35 @@ mod app_tests { }); } + /// Visual results come best first, skip rows the index cannot rate, and stop at the limit. + #[gpui::test] + fn ranked_matches_order_by_score_and_respect_the_limit(cx: &mut TestAppContext) { + let state = table(cx); + cx.update(|cx| { + state.update(cx, |state, _| { + let delegate = state.delegate_mut(); + delegate.set_image_paths(vec![ + Some("0.jpg".into()), + None, + Some("2.jpg".into()), + Some("3.jpg".into()), + ]); + let score = |path: &std::path::Path| match path.to_str() { + Some("0.jpg") => Some(0.2), + Some("2.jpg") => Some(0.9), + _ => None, + }; + let rows: Vec = delegate + .ranked_matches(score, 5) + .into_iter() + .map(|(row, _)| row) + .collect(); + assert_eq!(rows, vec![2, 0]); + assert_eq!(delegate.ranked_matches(score, 1).len(), 1); + }); + }); + } + /// A multi-row delete is one undo step, and the deleted rows come back where they were rather /// than all at the first index. #[gpui::test] diff --git a/crates/table/src/lib.rs b/crates/table/src/lib.rs index da46727f..2e82e5b0 100644 --- a/crates/table/src/lib.rs +++ b/crates/table/src/lib.rs @@ -20,6 +20,7 @@ mod note; mod panel; pub mod photos; mod row_index; +mod visual; pub use agent::{AGENT_SOURCE, respond_to_agent, respond_to_agent_async}; pub use delegate::{QrateTableDelegate, Selection, TableChanged}; diff --git a/crates/table/src/note.rs b/crates/table/src/note.rs index 3fba52b2..773a29ba 100644 --- a/crates/table/src/note.rs +++ b/crates/table/src/note.rs @@ -349,13 +349,24 @@ pub fn menu( let (rows, files) = { let delegate = table.read(cx).delegate(); let rows = target_rows(delegate, row); - let files = rows + let files: Vec = rows .iter() .filter_map(|&row| delegate.row_image(row).map(std::path::Path::to_path_buf)) .collect(); (rows, files) }; - selection_items(menu, &rows, files) + match (rows.len(), files.first()) { + (1, Some(file)) => { + let file = file.clone(); + menu.item( + PopupMenuItem::new("Find similar items").on_click(move |_, _, cx| { + crate::visual::find_similar(file.clone(), cx) + }), + ) + .separator() + } + _ => selection_items(menu, &rows, files), + } } None => menu, }; diff --git a/crates/table/src/panel.rs b/crates/table/src/panel.rs index a78c7e98..470d5d02 100644 --- a/crates/table/src/panel.rs +++ b/crates/table/src/panel.rs @@ -21,7 +21,7 @@ use crate::{ editing::{self, EditState}, floating::float_at, history::Cells, - note, photos, row_index, + note, photos, row_index, visual, }; const COLUMN_LAYOUT_KEY: &str = "table_columns"; @@ -95,6 +95,11 @@ pub struct TablePanel { _search_sub: Subscription, /// Linked documents being read for a search that includes them. `Some` while it runs. reading_documents: Option>, + /// The pending visual query. Replacing it cancels the one before, which debounces typing. + visual_query: Option>, + /// The file a visual search with an empty query ranks against, from "Find similar items". + similar: Option, + _visual_sub: Subscription, replace_input: Entity, replace_open: bool, _replace_sub: Subscription, @@ -369,6 +374,24 @@ impl TablePanel { }, ); + visual::init(cx); + // Picks up a "find similar" request from a menu, and re-runs a visual search as the model + // downloads and the index fills in. + let _visual_sub = cx.observe_global_in::(window, |this, window, cx| { + if let Some(path) = cx.global_mut::().similar.take() { + this.similar = Some(path); + this.search_opts.visual = true; + this.search_open = true; + this.search_input.update(cx, |input, cx| { + input.set_value("", window, cx); + input.focus(window, cx); + }); + this.refresh_search(cx); + } else if this.search_open && this.search_opts.visual { + this.refresh_search(cx); + } + }); + let _replace_sub = cx.subscribe(&replace_input, |this, _input, event: &InputEvent, cx| { if matches!(event, InputEvent::PressEnter { .. }) { this.replace(false, cx); @@ -395,6 +418,9 @@ impl TablePanel { search_error: false, _search_sub, reading_documents: None, + visual_query: None, + similar: None, + _visual_sub, replace_input, replace_open: false, _replace_sub, @@ -437,6 +463,10 @@ impl TablePanel { /// Recompute the find matches from the current query and jump to the first, if any. Called on /// every keystroke in the find bar (the scan is sub-millisecond for qrate's grids). fn refresh_search(&mut self, cx: &mut Context) { + if self.search_opts.visual { + self.search_visual(cx); + return; + } let needle = self.search_input.read(cx).value().to_string(); self.search_error = !needle.trim().is_empty() && compile_search(&needle, self.search_opts).is_none(); @@ -451,6 +481,49 @@ impl TablePanel { cx.notify(); } + /// Rank rows by how well their linked file matches the query, or looks like the file "Find + /// similar items" was raised on when the query is empty. Typing waits for a pause first. + fn search_visual(&mut self, cx: &mut Context) { + let files = self.state.read(cx).delegate().linked_files(); + visual::index(files, cx); + self.search_error = false; + let needle = self.search_input.read(cx).value().trim().to_string(); + let (query, pause) = match (needle.is_empty(), self.similar.clone()) { + (false, _) => ( + visual::Query::Text(needle), + std::time::Duration::from_millis(300), + ), + (true, Some(path)) => (visual::Query::Like(path), std::time::Duration::ZERO), + (true, None) => { + self.visual_query = None; + self.search_matches.clear(); + cx.notify(); + return; + } + }; + self.visual_query = Some(cx.spawn(async move |this, cx| { + cx.background_executor().timer(pause).await; + let Ok(score) = this.update(cx, |_, cx| visual::scorer(query, cx)) else { + return; + }; + let score = score.await; + this.update(cx, |this, cx| { + this.search_matches = score + .map(|score| { + this.state + .read(cx) + .delegate() + .ranked_matches(score, visual::RESULTS) + }) + .unwrap_or_default(); + this.search_ix = 0; + this.select_current_match(cx); + cx.notify(); + }) + .ok(); + })); + } + /// Read the text of every linked document the search has not seen yet, off the UI thread, then /// search again so their hits join the results. fn read_documents(&mut self, cx: &mut Context) { @@ -494,12 +567,15 @@ impl TablePanel { fn goto_match(&mut self, delta: isize, cx: &mut Context) { // Re-scan first: matches are view indices, and a filter change since last keystroke invalidates them. - let needle = self.search_input.read(cx).value().to_string(); - self.search_matches = self - .state - .read(cx) - .delegate() - .search_matches(&needle, self.search_opts); + // A visual ranking costs a model call, so it keeps the order it was given. + if !self.search_opts.visual { + let needle = self.search_input.read(cx).value().to_string(); + self.search_matches = self + .state + .read(cx) + .delegate() + .search_matches(&needle, self.search_opts); + } let n = self.search_matches.len(); if n == 0 { return; @@ -523,6 +599,9 @@ impl TablePanel { /// whole visible set instead, as a single undo step. Matches are cell-granular, so this /// substitutes every occurrence *within* each affected cell and leaves the rest of its text. fn replace(&mut self, all: bool, cx: &mut Context) { + if self.search_opts.visual { + return; + } let needle = self.search_input.read(cx).value().to_string(); let replacement = self.replace_input.read(cx).value().to_string(); let only = match all { @@ -922,7 +1001,15 @@ impl Render for TablePanel { fn render(&mut self, window: &mut Window, cx: &mut Context) -> impl IntoElement { let stripe = settings::effective_bool(crate::TABLE_STRIPES_KEY, cx); let query = self.search_input.read(cx).value(); - let count = if self.search_error { + let visual_status = visual::status(cx); + let count = if let Some(status) = self + .search_opts + .visual + .then(|| visual::status_label(&visual_status)) + .flatten() + { + status + } else if self.search_error { SharedString::from("Invalid regex") } else if !self.search_matches.is_empty() { SharedString::from(format!( @@ -931,7 +1018,10 @@ impl Render for TablePanel { self.search_matches.len() )) } else if query.trim().is_empty() { - SharedString::default() + match (&self.similar, self.search_opts.visual) { + (Some(_), true) => SharedString::from("No similar items"), + _ => SharedString::default(), + } } else if self.reading_documents.is_some() { SharedString::from("Reading files…") } else { @@ -1048,6 +1138,8 @@ impl Render for TablePanel { .ghost() .small() .selected(on) + // Text rules have no meaning for a query about what an image shows. + .disabled(opts.visual && id != "search-visual") .tooltip(tip) .map(|b| match icon { Some(icon) => b.icon(icon), @@ -1118,6 +1210,14 @@ impl Render for TablePanel { "Include text inside linked files", opts.files, |o| &mut o.files, + )) + .child(toggle( + "search-visual", + Some(IconName::Frame), + "", + "Search by what images show", + opts.visual, + |o| &mut o.visual, )), ) .child( @@ -1127,6 +1227,21 @@ impl Render for TablePanel { .text_color(muted) .child(count), ) + .when( + opts.visual + && matches!( + visual_status, + visual::Status::Missing | visual::Status::Failed(_) + ), + |bar| { + bar.child( + Button::new("visual-install") + .small() + .label(visual::download_label()) + .on_click(|_, _, cx| visual::install(cx)), + ) + }, + ) .child( Button::new("search-prev") .icon(IconName::ChevronUp) diff --git a/crates/table/src/visual.rs b/crates/table/src/visual.rs new file mode 100644 index 00000000..9efffde3 --- /dev/null +++ b/crates/table/src/visual.rs @@ -0,0 +1,306 @@ +//! Visual search for the open table: installing the model, keeping its index current for the rows' +//! linked files, and turning a query into ranked rows. +//! +//! Shared across windows as a global, because the model holds 600 MB and one copy is plenty. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use gpui::{App, Global, SharedString, Task}; +use visual_search::{Clip, Index, Stamp}; + +/// Files embedded per model call. Larger batches barely help on a CPU and delay the progress count. +const BATCH: usize = 16; + +/// How many rows a visual search returns, best first. +/// +/// ponytail: a fixed count, not a similarity threshold. CLIP scores cluster tightly, so a cut-off +/// needs tuning against real collections first. +pub(crate) const RESULTS: usize = 50; + +#[derive(Clone, PartialEq)] +pub(crate) enum Status { + Missing, + Downloading(u64), + Indexing { done: usize, total: usize }, + Ready, + Failed(SharedString), +} + +pub(crate) struct Visual { + pub status: Status, + clip: Option>, + index: Arc>, + /// Files the indexing job has already been handed this session, so asking again is free. + seen: HashSet, + job: Option>, + /// A "find similar" request waiting for the search bar to pick it up. + pub similar: Option, +} + +impl Global for Visual {} + +fn model_dir() -> Option { + Some( + settings::data_dir()? + .join("models") + .join(visual_search::MODEL_DIR), + ) +} + +fn index_file() -> Option { + Some(settings::data_dir()?.join("visual-index.bin")) +} + +pub(crate) fn init(cx: &mut App) { + if cx.has_global::() { + return; + } + let installed = model_dir().is_some_and(|dir| visual_search::installed(&dir)); + cx.set_global(Visual { + status: if installed { + Status::Ready + } else { + Status::Missing + }, + clip: None, + index: Arc::new(Mutex::new(Index::new(visual_search::MODEL))), + seen: HashSet::new(), + job: None, + similar: None, + }); +} + +/// For writes only: every call notifies the global's observers, which re-run the search bar. +fn state(cx: &mut App) -> &mut Visual { + cx.global_mut::() +} + +pub(crate) fn status(cx: &App) -> Status { + cx.global::().status.clone() +} + +/// What the search bar says instead of a match count while visual search cannot answer yet. +pub(crate) fn status_label(status: &Status) -> Option { + match status { + Status::Missing => Some("Model not installed".into()), + Status::Downloading(done) => Some( + format!( + "Downloading {}%", + done * 100 / visual_search::DOWNLOAD_SIZE.max(1) + ) + .into(), + ), + Status::Indexing { done, total } => Some(format!("Indexing {done} / {total}").into()), + Status::Ready => None, + Status::Failed(reason) => Some(reason.clone()), + } +} + +pub(crate) fn download_label() -> SharedString { + format!( + "Download model ({} MB)", + visual_search::DOWNLOAD_SIZE / 1_000_000 + ) + .into() +} + +/// Fetch the model, reporting progress into [`Status::Downloading`]. +pub(crate) fn install(cx: &mut App) { + let Some(dir) = model_dir() else { + return; + }; + if matches!(status(cx), Status::Downloading(_)) { + return; + } + state(cx).status = Status::Downloading(0); + let progress = Arc::new(AtomicU64::new(0)); + let finished = Arc::new(AtomicBool::new(false)); + let download = cx.background_executor().spawn({ + let (progress, finished) = (progress.clone(), finished.clone()); + async move { + let result = + visual_search::download(&dir, &|bytes| progress.store(bytes, Ordering::Relaxed)); + finished.store(true, Ordering::Relaxed); + result + } + }); + let job = cx.spawn(async move |cx| { + while !finished.load(Ordering::Relaxed) { + cx.background_executor() + .timer(Duration::from_millis(250)) + .await; + cx.update(|cx| { + state(cx).status = Status::Downloading(progress.load(Ordering::Relaxed)) + }); + } + let result = download.await; + cx.update(|cx| { + let visual = state(cx); + visual.job = None; + visual.status = match result { + Ok(()) => Status::Ready, + Err(err) => { + log::error!("could not download the visual search model: {err:#}"); + Status::Failed("Download failed".into()) + } + }; + }); + }); + state(cx).job = Some(job); +} + +/// Embed any of `paths` the index does not already hold for their current contents. Returns at +/// once; progress shows in [`Status::Indexing`]. +pub(crate) fn index(paths: Vec, cx: &mut App) { + let visual = cx.global::(); + if visual.status != Status::Ready || visual.job.is_some() { + return; + } + let fresh: Vec = paths + .into_iter() + .filter(|path| !visual.seen.contains(path)) + .collect(); + let (Some(dir), Some(file)) = (model_dir(), index_file()) else { + return; + }; + if fresh.is_empty() { + return; + } + let (index, clip) = (visual.index.clone(), visual.clip.clone()); + state(cx).seen.extend(fresh.iter().cloned()); + let job = cx.spawn(async move |cx| { + let prepared = cx + .background_executor() + .spawn({ + let index = index.clone(); + let file = file.clone(); + async move { + let clip = match clip { + Some(clip) => clip, + None => Arc::new(Clip::load(&dir).map_err(|err| format!("{err:#}"))?), + }; + let mut index = index.lock().unwrap_or_else(|err| err.into_inner()); + if index.is_empty() { + *index = Index::load(&file, visual_search::MODEL); + } + let stale: Vec<(PathBuf, Stamp)> = fresh + .into_iter() + .filter_map(|path| Stamp::of(&path).map(|stamp| (path, stamp))) + .filter(|(path, stamp)| !index.is_current(path, *stamp)) + .collect(); + Ok::<_, String>((clip, stale)) + } + }) + .await; + let (clip, stale) = match prepared { + Ok(prepared) => prepared, + Err(err) => { + log::error!("could not load the visual search model: {err}"); + cx.update(|cx| { + let visual = state(cx); + visual.job = None; + visual.status = Status::Failed("Model failed to load".into()); + }); + return; + } + }; + cx.update(|cx| state(cx).clip = Some(clip.clone())); + + let total = stale.len(); + for (at, batch) in stale.chunks(BATCH).enumerate() { + cx.update(|cx| { + state(cx).status = Status::Indexing { + done: at * BATCH, + total, + } + }); + let (clip, index, batch) = (clip.clone(), index.clone(), batch.to_vec()); + cx.background_executor() + .spawn(async move { embed_batch(&clip, &index, batch) }) + .await; + } + if total > 0 { + cx.background_executor() + .spawn(async move { + let index = index.lock().unwrap_or_else(|err| err.into_inner()); + if let Err(err) = index.save(&file) { + log::warn!( + "could not save the visual search index, it will be rebuilt next launch: {err}" + ); + } + }) + .await; + log::info!("visual search indexed {total} files"); + } + cx.update(|cx| { + let visual = state(cx); + visual.job = None; + visual.status = Status::Ready; + }); + }); + state(cx).job = Some(job); +} + +fn embed_batch(clip: &Clip, index: &Mutex, batch: Vec<(PathBuf, Stamp)>) { + let (files, images): (Vec<_>, Vec<_>) = batch + .into_iter() + .filter_map(|(path, stamp)| { + let pixels = preview::thumbnail_pixels(&path, preview::CARD, 0)?; + Some(((path, stamp), pixels)) + }) + .unzip(); + if images.is_empty() { + return; + } + match clip.embed_images(&images) { + Ok(vectors) => { + let mut index = index.lock().unwrap_or_else(|err| err.into_inner()); + for ((path, stamp), vector) in files.into_iter().zip(vectors) { + index.insert(path, stamp, vector); + } + } + Err(err) => log::warn!("visual search skipped {} files: {err:#}", files.len()), + } +} + +/// What the query should be compared against: a description, or another file's own vector. +pub(crate) enum Query { + Text(String), + Like(PathBuf), +} + +/// A scoring function over files for [`crate::delegate::QrateTableDelegate::ranked_matches`], or +/// `None` when the model is not loaded or the query cannot be embedded. +pub(crate) fn scorer( + query: Query, + cx: &App, +) -> Task Option + use<>>> { + let visual = cx.global::(); + let (clip, index) = (visual.clip.clone(), visual.index.clone()); + cx.background_executor().spawn(async move { + let vector = match query { + Query::Text(text) => clip? + .embed_text(&text) + .map_err(|err| log::warn!("could not embed a visual search query: {err:#}")) + .ok()?, + Query::Like(path) => index + .lock() + .unwrap_or_else(|err| err.into_inner()) + .vector(&path)? + .to_vec(), + }; + Some(move |path: &Path| { + let index = index.lock().unwrap_or_else(|err| err.into_inner()); + Some(visual_search::similarity(&vector, index.vector(path)?)) + }) + }) +} + +/// Ask the search bar to show the files that look most like `path`. +pub(crate) fn find_similar(path: PathBuf, cx: &mut App) { + state(cx).similar = Some(path); +} diff --git a/crates/visual-search/Cargo.toml b/crates/visual-search/Cargo.toml new file mode 100644 index 00000000..a0ba7649 --- /dev/null +++ b/crates/visual-search/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "visual-search" +version.workspace = true +edition.workspace = true + +[dependencies] +anyhow.workspace = true +log.workspace = true +serde_json.workspace = true +# Pure Rust, so the model runs on every platform CI builds. Held at 0.9: from 0.10 candle-core +# pulls in `tokenizers` and its C oniguruma build, which nothing here calls. +candle-core = "0.9.2" +candle-nn = "0.9.2" +candle-transformers = "0.9.2" +image = { version = "0.25", default-features = false } +regex = "1" +reqwest = { version = "0.12", features = ["blocking"] } +sha2 = "0.10" + +[dev-dependencies] +dirs.workspace = true diff --git a/crates/visual-search/src/index.rs b/crates/visual-search/src/index.rs new file mode 100644 index 00000000..3a8fa223 --- /dev/null +++ b/crates/visual-search/src/index.rs @@ -0,0 +1,189 @@ +//! Every embedded file's vector, kept on disk between launches. +//! +//! Keyed by path and stamped with the file's size and modification time, so an edited or replaced +//! file is embedded again and an untouched one never is. +//! +//! ponytail: entries for deleted files are never dropped, at 2 KB each. Prune on save if a +//! long-lived index grows noticeably. + +use std::collections::HashMap; +use std::fs; +use std::io::{self, BufReader, BufWriter, Read, Write}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, UNIX_EPOCH}; + +const MAGIC: &[u8; 8] = b"qrvidx01"; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Stamp { + len: u64, + modified: Duration, +} + +impl Stamp { + pub fn of(path: &Path) -> Option { + let meta = fs::metadata(path).ok()?; + Some(Self { + len: meta.len(), + modified: meta.modified().ok()?.duration_since(UNIX_EPOCH).ok()?, + }) + } +} + +#[derive(Default)] +pub struct Index { + model: String, + entries: HashMap)>, +} + +impl Index { + pub fn new(model: &str) -> Self { + Self { + model: model.to_owned(), + entries: HashMap::new(), + } + } + + /// The index saved at `file`, or an empty one when there is none, it is unreadable, or it was + /// built by a different model. + pub fn load(file: &Path, model: &str) -> Self { + match read(file) { + Ok(index) if index.model == model => index, + Ok(_) => { + log::info!("the visual search index was built by another model, rebuilding it"); + Self::new(model) + } + Err(err) if err.kind() == io::ErrorKind::NotFound => Self::new(model), + Err(err) => { + log::warn!("could not read the visual search index, rebuilding it: {err}"); + Self::new(model) + } + } + } + + /// Written beside `file` and renamed over it, so a crash mid-write leaves the previous index. + pub fn save(&self, file: &Path) -> io::Result<()> { + let partial = file.with_extension("part"); + let mut out = BufWriter::new(fs::File::create(&partial)?); + out.write_all(MAGIC)?; + write_bytes(&mut out, self.model.as_bytes())?; + for (path, (stamp, vector)) in &self.entries { + let Some(path) = path.to_str() else { + continue; + }; + write_bytes(&mut out, path.as_bytes())?; + out.write_all(&stamp.len.to_le_bytes())?; + out.write_all(&stamp.modified.as_secs().to_le_bytes())?; + out.write_all(&stamp.modified.subsec_nanos().to_le_bytes())?; + out.write_all(&(vector.len() as u32).to_le_bytes())?; + for value in vector { + out.write_all(&value.to_le_bytes())?; + } + } + out.into_inner() + .map_err(|err| err.into_error())? + .sync_all()?; + fs::rename(partial, file) + } + + pub fn is_current(&self, path: &Path, stamp: Stamp) -> bool { + self.entries + .get(path) + .is_some_and(|(kept, _)| *kept == stamp) + } + + pub fn insert(&mut self, path: PathBuf, stamp: Stamp, vector: Vec) { + self.entries.insert(path, (stamp, vector)); + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub fn vector(&self, path: &Path) -> Option<&[f32]> { + self.entries.get(path).map(|(_, vector)| vector.as_slice()) + } +} + +fn read(file: &Path) -> io::Result { + let mut input = BufReader::new(fs::File::open(file)?); + let mut magic = [0; 8]; + input.read_exact(&mut magic)?; + if &magic != MAGIC { + return Err(io::Error::new(io::ErrorKind::InvalidData, "not an index")); + } + let model = String::from_utf8(read_bytes(&mut input)?) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + let mut index = Index::new(&model); + loop { + let path = match read_bytes(&mut input) { + Ok(path) => path, + Err(err) if err.kind() == io::ErrorKind::UnexpectedEof => return Ok(index), + Err(err) => return Err(err), + }; + let path = String::from_utf8(path) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + let len = u64::from_le_bytes(read_array(&mut input)?); + let secs = u64::from_le_bytes(read_array(&mut input)?); + let nanos = u32::from_le_bytes(read_array(&mut input)?); + let dim = u32::from_le_bytes(read_array(&mut input)?) as usize; + let vector = (0..dim) + .map(|_| read_array(&mut input).map(f32::from_le_bytes)) + .collect::>()?; + let stamp = Stamp { + len, + modified: Duration::new(secs, nanos), + }; + index.insert(path.into(), stamp, vector); + } +} + +fn write_bytes(out: &mut impl Write, bytes: &[u8]) -> io::Result<()> { + out.write_all(&(bytes.len() as u32).to_le_bytes())?; + out.write_all(bytes) +} + +fn read_bytes(input: &mut impl Read) -> io::Result> { + let len = u32::from_le_bytes(read_array(input)?) as usize; + let mut bytes = vec![0; len]; + input.read_exact(&mut bytes)?; + Ok(bytes) +} + +fn read_array(input: &mut impl Read) -> io::Result<[u8; N]> { + let mut bytes = [0; N]; + input.read_exact(&mut bytes)?; + Ok(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trips_and_forgets_another_models_vectors() { + let dir = std::env::temp_dir().join("qrate-visual-index-test"); + fs::create_dir_all(&dir).unwrap(); + let file = dir.join("index.bin"); + let photo = dir.join("photo.jpg"); + fs::write(&photo, b"pixels").unwrap(); + let stamp = Stamp::of(&photo).unwrap(); + + let mut index = Index::new("clip-a"); + index.insert(photo.clone(), stamp, vec![0.6, 0.8]); + index.save(&file).unwrap(); + + let loaded = Index::load(&file, "clip-a"); + assert!(loaded.is_current(&photo, stamp)); + assert_eq!(loaded.vector(&photo), Some(&[0.6, 0.8][..])); + + fs::write(&photo, b"a different, longer set of pixels").unwrap(); + assert!( + !loaded.is_current(&photo, Stamp::of(&photo).unwrap()), + "an edited file is stale" + ); + assert!(Index::load(&file, "clip-b").vector(&photo).is_none()); + + let _ = fs::remove_dir_all(&dir); + } +} diff --git a/crates/visual-search/src/lib.rs b/crates/visual-search/src/lib.rs new file mode 100644 index 00000000..dfb52d53 --- /dev/null +++ b/crates/visual-search/src/lib.rs @@ -0,0 +1,206 @@ +//! Searching a collection by what its images show. +//! +//! CLIP turns an image and a sentence into vectors in the same space, so "a sawmill by the water" +//! lands near photographs of one. Every linked file's thumbnail is embedded once into an [`Index`] +//! on disk; a query is embedded as text, or taken from another image for "find similar", and rows +//! are ranked by how close their vectors are. +//! +//! The model runs on the CPU through candle, which is pure Rust. Its weights are not part of the +//! install: [`download`] fetches a pinned revision and checks it against the published checksum. + +mod index; +mod tokenizer; + +use std::fs; +use std::io::{Read as _, Write as _}; +use std::path::Path; + +use anyhow::{Context as _, Result, ensure}; +use candle_core::{DType, Device, Tensor}; +use candle_transformers::models::clip::{ClipConfig, ClipModel, div_l2_norm}; +use image::{DynamicImage, RgbaImage, imageops::FilterType}; +use sha2::{Digest as _, Sha256}; + +pub use index::{Index, Stamp}; + +/// Which weights an index was built with. Vectors from different models are not comparable, so an +/// index recorded under another name is discarded rather than searched. +pub const MODEL: &str = "openai/clip-vit-base-patch32@b33cedf"; + +/// Folder name for the weights under qrate's models directory. +pub const MODEL_DIR: &str = "clip-vit-base-patch32"; + +const REVISION: &str = "b33cedfd0df4e43b8238760678fcc89e1a0d38b3"; + +struct File { + name: &'static str, + size: u64, + sha256: &'static str, +} + +const FILES: [File; 2] = [ + File { + name: "tokenizer.json", + size: 2_224_041, + sha256: "b556ac8c99757ffb677208af34bc8c6721572114111a6e0aaf5fa69ff0b8d842", + }, + File { + name: "model.safetensors", + size: 605_157_884, + sha256: "99d28a652e6ec46629ab7047a0ac82c69b1fe11e0ce672c43af65d3a9a3fc05d", + }, +]; + +/// Bytes [`download`] fetches in total, for the install prompt. +pub const DOWNLOAD_SIZE: u64 = FILES[0].size + FILES[1].size; + +/// Whether the weights are in `dir`. Checks sizes, not checksums: [`download`] only renames a file +/// into place after its checksum matched. +pub fn installed(dir: &Path) -> bool { + FILES + .iter() + .all(|file| fs::metadata(dir.join(file.name)).is_ok_and(|meta| meta.len() == file.size)) +} + +/// Fetch the pinned weights into `dir`, calling `progress` with bytes done so far. A file is written +/// beside its final name and moved into place only once its SHA-256 matches, so an interrupted or +/// tampered download never looks installed. +pub fn download(dir: &Path, progress: &dyn Fn(u64)) -> Result<()> { + fs::create_dir_all(dir).with_context(|| format!("could not create {}", dir.display()))?; + // No overall timeout: the weights are 600 MB and a slow connection is not a failure. + let client = reqwest::blocking::Client::builder().timeout(None).build()?; + let mut done = 0; + for file in &FILES { + let target = dir.join(file.name); + if fs::metadata(&target).is_ok_and(|meta| meta.len() == file.size) { + done += file.size; + progress(done); + continue; + } + let url = format!( + "https://huggingface.co/openai/clip-vit-base-patch32/resolve/{REVISION}/{}", + file.name + ); + let mut response = client.get(&url).send()?.error_for_status()?; + let partial = target.with_extension("part"); + let mut out = fs::File::create(&partial)?; + let mut hasher = Sha256::new(); + let mut buffer = vec![0; 1 << 16]; + loop { + let read = response.read(&mut buffer)?; + if read == 0 { + break; + } + out.write_all(&buffer[..read])?; + hasher.update(&buffer[..read]); + done += read as u64; + progress(done); + } + drop(out); + let digest = format!("{:x}", hasher.finalize()); + if digest != file.sha256 { + let _ = fs::remove_file(&partial); + } + ensure!( + digest == file.sha256, + "the downloaded {} did not match its published checksum", + file.name + ); + fs::rename(&partial, &target)?; + } + Ok(()) +} + +/// The loaded model. Holds about 600 MB, memory-mapped from the weights file. +pub struct Clip { + model: ClipModel, + tokenizer: tokenizer::Tokenizer, +} + +/// The normalisation CLIP was trained with, per RGB channel. +const MEAN: [f32; 3] = [0.481_454_66, 0.457_827_5, 0.408_210_73]; +const STD: [f32; 3] = [0.268_629_54, 0.261_302_6, 0.275_777_1]; + +impl Clip { + pub fn load(dir: &Path) -> Result { + let tokenizer = tokenizer::Tokenizer::load(&dir.join("tokenizer.json"))?; + // SAFETY: the weights file is only ever replaced by rename, never modified in place, so the + // mapping cannot change underneath the model. + let weights = unsafe { + candle_nn::VarBuilder::from_mmaped_safetensors( + &[dir.join("model.safetensors")], + DType::F32, + &Device::Cpu, + )? + }; + let model = ClipModel::new(weights, &ClipConfig::vit_base_patch32())?; + Ok(Self { model, tokenizer }) + } + + /// One unit-length vector per image, in order. + pub fn embed_images(&self, images: &[RgbaImage]) -> Result>> { + let size = ClipConfig::vit_base_patch32().image_size; + let mut pixels = Vec::with_capacity(images.len() * 3 * size * size); + for image in images { + // Fill then centre-crop, as CLIP's own preprocessing does, so nothing is squashed. + let rgb = DynamicImage::ImageRgba8(image.clone()) + .resize_to_fill(size as u32, size as u32, FilterType::CatmullRom) + .to_rgb8(); + for channel in 0..3 { + pixels.extend( + rgb.pixels().map(|px| { + (f32::from(px.0[channel]) / 255.0 - MEAN[channel]) / STD[channel] + }), + ); + } + } + let batch = Tensor::from_vec(pixels, (images.len(), 3, size, size), &Device::Cpu)?; + Ok(div_l2_norm(&self.model.get_image_features(&batch)?)?.to_vec2()?) + } + + /// A unit-length vector for a description of what an image shows. + pub fn embed_text(&self, text: &str) -> Result> { + let ids = Tensor::new(self.tokenizer.encode(text), &Device::Cpu)?.unsqueeze(0)?; + let features = div_l2_norm(&self.model.get_text_features(&ids)?)?; + Ok(features.squeeze(0)?.to_vec1()?) + } +} + +/// Cosine similarity of two unit-length vectors. +pub fn similarity(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b).map(|(a, b)| a * b).sum() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Runs against the real weights when they are installed. Without them this is the skip path, + /// not a pass — `download` into the models folder first. + #[test] + fn a_description_finds_the_image_it_describes() { + let Some(dir) = dirs::data_local_dir().map(|d| d.join("qrate/models").join(MODEL_DIR)) + else { + return; + }; + if !installed(&dir) { + eprintln!("skipping the model check: CLIP weights are not installed"); + return; + } + let clip = Clip::load(&dir).unwrap(); + let red = RgbaImage::from_pixel(64, 64, image::Rgba([220, 20, 20, 255])); + let blue = RgbaImage::from_pixel(64, 64, image::Rgba([20, 40, 220, 255])); + let images = clip.embed_images(&[red, blue]).unwrap(); + assert_eq!(images[0].len(), 512); + + let query = clip.embed_text("a plain red square").unwrap(); + assert!( + (similarity(&query, &query) - 1.0).abs() < 1e-3, + "unit length" + ); + assert!( + similarity(&query, &images[0]) > similarity(&query, &images[1]), + "red ranks above blue" + ); + } +} diff --git a/crates/visual-search/src/tokenizer.rs b/crates/visual-search/src/tokenizer.rs new file mode 100644 index 00000000..bb248f7d --- /dev/null +++ b/crates/visual-search/src/tokenizer.rs @@ -0,0 +1,135 @@ +//! CLIP's byte-level BPE, read from the model's own `tokenizer.json`. +//! +//! ponytail: skips the NFC normalisation step, which only changes text with decomposed accents. +//! Add `unicode-normalization` if queries in such text rank oddly. + +use std::collections::HashMap; +use std::path::Path; + +use anyhow::{Context as _, Result}; +use regex::Regex; + +const START: u32 = 49406; +const END: u32 = 49407; +/// The text encoder's position embeddings stop here, start and end tokens included. +const CONTEXT: usize = 77; + +pub struct Tokenizer { + vocab: HashMap, + ranks: HashMap<(String, String), usize>, + split: Regex, + bytes: [char; 256], +} + +impl Tokenizer { + pub fn load(file: &Path) -> Result { + let json: serde_json::Value = serde_json::from_slice(&std::fs::read(file)?)?; + let model = &json["model"]; + let vocab = serde_json::from_value(model["vocab"].clone()).context("tokenizer vocab")?; + let ranks = model["merges"] + .as_array() + .context("tokenizer merges")? + .iter() + .filter_map(|merge| merge.as_str()?.split_once(' ')) + .enumerate() + .map(|(rank, (a, b))| ((a.to_owned(), b.to_owned()), rank)) + .collect(); + Ok(Self { + vocab, + ranks, + split: Regex::new(r"'s|'t|'re|'ve|'m|'ll|'d|\p{L}+|\p{N}|[^\s\p{L}\p{N}]+")?, + bytes: byte_chars(), + }) + } + + pub fn encode(&self, text: &str) -> Vec { + let text = text + .split_whitespace() + .collect::>() + .join(" ") + .to_lowercase(); + let mut ids = vec![START]; + for word in self.split.find_iter(&text) { + let symbols = word + .as_str() + .bytes() + .map(|b| self.bytes[b as usize].to_string()); + ids.extend( + self.merge(symbols.collect()) + .iter() + .map(|symbol| self.vocab.get(symbol).copied().unwrap_or(END)), + ); + } + ids.truncate(CONTEXT - 1); + ids.push(END); + ids + } + + /// Apply merges lowest rank first until none apply, marking the word's end the way the + /// vocabulary does. + fn merge(&self, mut symbols: Vec) -> Vec { + if let Some(last) = symbols.last_mut() { + last.push_str(""); + } + loop { + let best = symbols + .windows(2) + .enumerate() + .filter_map(|(at, pair)| { + let rank = self.ranks.get(&(pair[0].clone(), pair[1].clone()))?; + Some((*rank, at)) + }) + .min(); + let Some((_, at)) = best else { + return symbols; + }; + let right = symbols.remove(at + 1); + symbols[at].push_str(&right); + } + } +} + +/// GPT-2's reversible byte-to-character table: printable bytes stand for themselves, the rest are +/// shifted past 255 so every byte has a visible, vocabulary-safe character. +fn byte_chars() -> [char; 256] { + let mut table = ['\0'; 256]; + let mut shifted = 0; + for byte in 0..=255u8 { + table[byte as usize] = match byte { + b'!'..=b'~' | 0xA1..=0xAC | 0xAE..=0xFF => char::from(byte), + _ => { + shifted += 1; + char::from_u32(255 + shifted).unwrap_or('\0') + } + }; + } + table +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unprintable_bytes_move_past_latin_one() { + let table = byte_chars(); + assert_eq!(table[b'a' as usize], 'a'); + assert_eq!(table[b' ' as usize], '\u{120}', "GPT-2 writes a space as Ġ"); + } + + #[test] + fn encodes_like_the_reference_tokenizer() { + let Some(file) = dirs::data_local_dir() + .map(|d| d.join("qrate/models/clip-vit-base-patch32/tokenizer.json")) + .filter(|file| file.is_file()) + else { + eprintln!("skipping the tokenizer check: CLIP weights are not installed"); + return; + }; + let tokenizer = Tokenizer::load(&file).unwrap(); + assert_eq!( + tokenizer.encode("A photo of a CAT"), + vec![START, 320, 1125, 539, 320, 2368, END] + ); + } +} From f8c2026a3496314b4ffc2bcda919513987189e92 Mon Sep 17 00:00:00 2001 From: devnull03 <56480041+devnull03@users.noreply.github.com> Date: Sun, 13 Sep 2026 16:59:55 -0700 Subject: [PATCH 03/12] fix(table): keep disk work out of the visual index lock Stat linked files and load the saved index before taking the lock the search bar ranks through, and save the index every 40 batches so quitting part-way through a first index keeps the work. --- crates/table/src/visual.rs | 58 +++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/crates/table/src/visual.rs b/crates/table/src/visual.rs index 9efffde3..dd411620 100644 --- a/crates/table/src/visual.rs +++ b/crates/table/src/visual.rs @@ -15,6 +15,9 @@ use visual_search::{Clip, Index, Stamp}; /// Files embedded per model call. Larger batches barely help on a CPU and delay the progress count. const BATCH: usize = 16; +/// Batches between index saves, about a minute of CPU work, so quitting mid-index keeps progress. +const SAVE_EVERY: usize = 40; + /// How many rows a visual search returns, best first. /// /// ponytail: a fixed count, not a similarity threshold. CLIP scores cluster tightly, so a cut-off @@ -183,13 +186,19 @@ pub(crate) fn index(paths: Vec, cx: &mut App) { Some(clip) => clip, None => Arc::new(Clip::load(&dir).map_err(|err| format!("{err:#}"))?), }; - let mut index = index.lock().unwrap_or_else(|err| err.into_inner()); - if index.is_empty() { - *index = Index::load(&file, visual_search::MODEL); - } - let stale: Vec<(PathBuf, Stamp)> = fresh + // Stat outside the lock: searches rank rows on the UI thread through this index. + let stamped: Vec<(PathBuf, Stamp)> = fresh .into_iter() .filter_map(|path| Stamp::of(&path).map(|stamp| (path, stamp))) + .collect(); + let empty = lock(&index).is_empty(); + let loaded = empty.then(|| Index::load(&file, visual_search::MODEL)); + let mut index = lock(&index); + if let Some(loaded) = loaded { + *index = loaded; + } + let stale: Vec<(PathBuf, Stamp)> = stamped + .into_iter() .filter(|(path, stamp)| !index.is_current(path, *stamp)) .collect(); Ok::<_, String>((clip, stale)) @@ -218,22 +227,19 @@ pub(crate) fn index(paths: Vec, cx: &mut App) { total, } }); - let (clip, index, batch) = (clip.clone(), index.clone(), batch.to_vec()); - cx.background_executor() - .spawn(async move { embed_batch(&clip, &index, batch) }) - .await; - } - if total > 0 { + let (clip, index, file, batch) = + (clip.clone(), index.clone(), file.clone(), batch.to_vec()); + let last = (at + 1) * BATCH >= total; cx.background_executor() .spawn(async move { - let index = index.lock().unwrap_or_else(|err| err.into_inner()); - if let Err(err) = index.save(&file) { - log::warn!( - "could not save the visual search index, it will be rebuilt next launch: {err}" - ); + embed_batch(&clip, &index, batch); + if last || at % SAVE_EVERY == SAVE_EVERY - 1 { + save(&index, &file); } }) .await; + } + if total > 0 { log::info!("visual search indexed {total} files"); } cx.update(|cx| { @@ -245,6 +251,16 @@ pub(crate) fn index(paths: Vec, cx: &mut App) { state(cx).job = Some(job); } +fn lock(index: &Mutex) -> std::sync::MutexGuard<'_, Index> { + index.lock().unwrap_or_else(|err| err.into_inner()) +} + +fn save(index: &Mutex, file: &Path) { + if let Err(err) = lock(index).save(file) { + log::warn!("could not save the visual search index, it will be rebuilt next launch: {err}"); + } +} + fn embed_batch(clip: &Clip, index: &Mutex, batch: Vec<(PathBuf, Stamp)>) { let (files, images): (Vec<_>, Vec<_>) = batch .into_iter() @@ -258,7 +274,7 @@ fn embed_batch(clip: &Clip, index: &Mutex, batch: Vec<(PathBuf, Stamp)>) } match clip.embed_images(&images) { Ok(vectors) => { - let mut index = index.lock().unwrap_or_else(|err| err.into_inner()); + let mut index = lock(index); for ((path, stamp), vector) in files.into_iter().zip(vectors) { index.insert(path, stamp, vector); } @@ -287,14 +303,10 @@ pub(crate) fn scorer( .embed_text(&text) .map_err(|err| log::warn!("could not embed a visual search query: {err:#}")) .ok()?, - Query::Like(path) => index - .lock() - .unwrap_or_else(|err| err.into_inner()) - .vector(&path)? - .to_vec(), + Query::Like(path) => lock(&index).vector(&path)?.to_vec(), }; Some(move |path: &Path| { - let index = index.lock().unwrap_or_else(|err| err.into_inner()); + let index = lock(&index); Some(visual_search::similarity(&vector, index.vector(path)?)) }) }) From 3c75c9b57c7be6f3234c71e0a221f2442c8c8763 Mon Sep 17 00:00:00 2001 From: devnull03 <56480041+devnull03@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:33:08 -0700 Subject: [PATCH 04/12] fix(table): stop the visual search observer re-triggering itself Taking the find-similar request through global_mut notified the same observer again, spinning the UI thread once indexing touched the global. Also gives the visual toggle a picture icon. --- assets/icons/image.svg | 1 + crates/app/src/assets.rs | 10 +++++++--- crates/table/src/panel.rs | 15 +++++++++------ 3 files changed, 17 insertions(+), 9 deletions(-) create mode 100644 assets/icons/image.svg diff --git a/assets/icons/image.svg b/assets/icons/image.svg new file mode 100644 index 00000000..135ea222 --- /dev/null +++ b/assets/icons/image.svg @@ -0,0 +1 @@ + diff --git a/crates/app/src/assets.rs b/crates/app/src/assets.rs index a626bc69..4b0b17fc 100644 --- a/crates/app/src/assets.rs +++ b/crates/app/src/assets.rs @@ -1,6 +1,6 @@ //! Our own icons in front of `gpui_component_assets`, which is otherwise the only asset source. -//! Lucide ships no filled panel glyph, so the title bar's "this dock is open" state needs three -//! SVGs of our own. +//! The bundled icon set has no filled panel glyphs (the title bar's "this dock is open" state) and +//! no picture glyph (visual search), so those SVGs are ours, copied from Lucide. use std::borrow::Cow; @@ -8,7 +8,7 @@ use gpui::{AssetSource, Result, SharedString}; pub struct Assets; -const OWN: [(&str, &str); 3] = [ +const OWN: [(&str, &str); 4] = [ ( "icons/panel-left-filled.svg", include_str!("../../../assets/icons/panel-left-filled.svg"), @@ -21,6 +21,10 @@ const OWN: [(&str, &str); 3] = [ "icons/panel-bottom-filled.svg", include_str!("../../../assets/icons/panel-bottom-filled.svg"), ), + ( + "icons/image.svg", + include_str!("../../../assets/icons/image.svg"), + ), ]; impl AssetSource for Assets { diff --git a/crates/table/src/panel.rs b/crates/table/src/panel.rs index 470d5d02..f90caceb 100644 --- a/crates/table/src/panel.rs +++ b/crates/table/src/panel.rs @@ -3,7 +3,7 @@ use std::ops::RangeInclusive; use gpui::prelude::FluentBuilder as _; use gpui::*; use gpui_component::{ - ActiveTheme, Disableable as _, IconName, Selectable as _, Sizable as _, + ActiveTheme, Disableable as _, Icon, IconName, Selectable as _, Sizable as _, button::{Button, ButtonVariants as _}, h_flex, input::{Escape, Input, InputEvent, InputState, TextareaState}, @@ -378,7 +378,10 @@ impl TablePanel { // Picks up a "find similar" request from a menu, and re-runs a visual search as the model // downloads and the index fills in. let _visual_sub = cx.observe_global_in::(window, |this, window, cx| { - if let Some(path) = cx.global_mut::().similar.take() { + // Read first: `global_mut` notifies this observer again, so calling it unconditionally loops. + if cx.global::().similar.is_some() + && let Some(path) = cx.global_mut::().similar.take() + { this.similar = Some(path); this.search_opts.visual = true; this.search_open = true; @@ -1129,7 +1132,7 @@ impl Render for TablePanel { .when(self.search_open, |this| { let opts = self.search_opts; let toggle = |id: &'static str, - icon: Option, + icon: Option, label: &'static str, tip: &'static str, on: bool, @@ -1181,7 +1184,7 @@ impl Render for TablePanel { .child(div().flex_1().child(Input::new(&self.search_input))) .child(toggle( "search-case", - Some(IconName::CaseSensitive), + Some(IconName::CaseSensitive.into()), "", "Match case", opts.case, @@ -1205,7 +1208,7 @@ impl Render for TablePanel { )) .child(toggle( "search-files", - Some(IconName::BookOpen), + Some(IconName::BookOpen.into()), "", "Include text inside linked files", opts.files, @@ -1213,7 +1216,7 @@ impl Render for TablePanel { )) .child(toggle( "search-visual", - Some(IconName::Frame), + Some(Icon::empty().path("icons/image.svg")), "", "Search by what images show", opts.visual, From 19108e764c9b76d28f866777bc2161e3267816c0 Mon Sep 17 00:00:00 2001 From: devnull03 <56480041+devnull03@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:49:46 -0700 Subject: [PATCH 05/12] feat(table): narrow the view to search hits, in table and gallery A visual or linked-file search now shows only its hit rows, visual ones best first, until the search is cleared or closed. A breadth slider replaces the fixed top 50 with a cut-off relative to the best hit. The find bar is drawn by the views panel, so it stays up and keeps its search across Table and Gallery, and the gallery scrolls to follow the selected hit. Replace is disabled during a visual search. Linked-file text is now opt-in, since turning it on filters. --- crates/app/src/actions.rs | 10 +- crates/table/src/delegate.rs | 117 ++++-- crates/table/src/panel.rs | 524 +++++++++++++++----------- crates/table/src/visual.rs | 44 ++- crates/workspace/src/views/gallery.rs | 2 + crates/workspace/src/views/mod.rs | 76 +++- 6 files changed, 501 insertions(+), 272 deletions(-) diff --git a/crates/app/src/actions.rs b/crates/app/src/actions.rs index 29e9dadc..aa5d4764 100644 --- a/crates/app/src/actions.rs +++ b/crates/app/src/actions.rs @@ -50,11 +50,15 @@ pub fn key_bindings() -> Vec { // registered globally in `main.rs`, so this only adds the key. KeyBinding::new("secondary-,", OpenSettings, None), // The grid's own actions are declared in `crate::table` (app→table is one-way). Ctrl+F is - // scoped to the panel so it works wherever focus sits inside it; everything below acts on a + // scoped to the centre views so it works in Table and Gallery alike; everything below acts on a // grid selection and is scoped to the grid, which keeps the cell editor's own Ctrl+Z/Ctrl+C // and its Enter-to-commit intact while it has focus. - KeyBinding::new("secondary-f", table::Search, Some("TablePanel")), - KeyBinding::new("secondary-h", table::Replace, Some("TablePanel")), + KeyBinding::new("secondary-f", table::Search, Some(workspace::VIEWS_CONTEXT)), + KeyBinding::new( + "secondary-h", + table::Replace, + Some(workspace::VIEWS_CONTEXT), + ), KeyBinding::new("enter", table::EditCell, Some(table::GRID_CONTEXT)), KeyBinding::new("secondary-z", table::Undo, Some(table::GRID_CONTEXT)), KeyBinding::new("secondary-y", table::Redo, Some(table::GRID_CONTEXT)), diff --git a/crates/table/src/delegate.rs b/crates/table/src/delegate.rs index e7fe4ded..583d82da 100644 --- a/crates/table/src/delegate.rs +++ b/crates/table/src/delegate.rs @@ -79,6 +79,11 @@ pub struct QrateTableDelegate { /// View→source row mapping: `visible_rows[view] == source`. The library only ever sees this /// narrowed set, so filtering composes with the virtualized render for free. visible_rows: Vec, + /// The rows the column filters let through, before a search narrows them. What searches scan, + /// so typing more can widen the hits again. + filtered_rows: Vec, + /// A visual or linked-file search's hits as source rows, in the order the view shows them. + search_rows: Option>, /// Per-data-column set of *excluded* cell values, parallel to `columns`. Empty = no filter. filters: Vec>, /// Whether each column offers a filter dropdown at all, parallel to `columns`. Off for every @@ -117,6 +122,8 @@ impl QrateTableDelegate { image_paths: Vec::new(), document_text: HashMap::new(), visible_rows: Vec::new(), + filtered_rows: Vec::new(), + search_rows: None, filters: Vec::new(), filters_enabled: Vec::new(), subdelimiter: SharedString::default(), @@ -201,7 +208,17 @@ impl QrateTableDelegate { } fn recompute_visible(&mut self) { - self.visible_rows = compute_visible_rows(&self.rows, &self.filters, &self.subdelimiter); + self.filtered_rows = compute_visible_rows(&self.rows, &self.filters, &self.subdelimiter); + self.visible_rows = match &self.search_rows { + Some(hits) => { + let kept: HashSet = self.filtered_rows.iter().copied().collect(); + hits.iter() + .copied() + .filter(|row| kept.contains(row)) + .collect() + } + None => self.filtered_rows.clone(), + }; // `range` is in view coordinates, which this just redefined. self.range = None; } @@ -347,29 +364,64 @@ impl QrateTableDelegate { hits } - /// The visible rows with a linked file that `score` can rate, best first and at most `limit` of - /// them, addressed like search hits. - pub(crate) fn ranked_matches( + /// Every filtered row with a linked file that `score` can rate, as `(score, source_row, + /// data_col)` best first. Ignores any narrowing a previous search applied. + pub(crate) fn ranked_rows( &self, score: impl Fn(&Path) -> Option, - limit: usize, - ) -> Vec<(usize, usize)> { + ) -> Vec<(f32, usize, usize)> { let mut scored: Vec<(f32, usize, usize)> = self - .visible_rows + .filtered_rows .iter() - .enumerate() - .filter_map(|(view, &source)| { + .filter_map(|&source| { let path = self.image_paths.get(source)?.as_ref()?; let col = file_column(self.rows.get(source)?, path); - Some((score(path)?, view, col)) + Some((score(path)?, source, col)) }) .collect(); scored.sort_by(|a, b| b.0.total_cmp(&a.0)); scored + } + + /// The filtered rows with a hit for `needle`, in cells or linked files, as source rows in order. + /// Ignores any narrowing a previous search applied. + pub(crate) fn hit_rows(&self, needle: &str, opts: SearchOpts) -> Vec { + let mut hits = find_matches( + &self.rows, + &self.filtered_rows, + self.columns.len(), + needle, + opts, + ); + if opts.files { + hits.extend(find_file_matches( + &self.rows, + &self.filtered_rows, + &self.image_paths, + &self.document_text, + needle, + opts, + )); + } + let mut rows: Vec = hits .into_iter() - .take(limit) - .map(|(_, view, col)| (view, col)) - .collect() + .map(|(at, _)| self.filtered_rows[at]) + .collect(); + rows.sort_unstable(); + rows.dedup(); + rows + } + + /// Show only `rows`, in that order, until a later call passes `None`. Column filters still apply. + /// Returns whether the view changed. + pub(crate) fn set_search_rows(&mut self, rows: Option>) -> bool { + if self.search_rows == rows { + return false; + } + self.search_rows = rows; + self.recompute_visible(); + self.editing = EditState::Idle; + true } /// Every distinct file the rows link to. @@ -456,7 +508,9 @@ impl QrateTableDelegate { self.history = History::default(); self.filters = vec![HashSet::new(); self.columns.len()]; self.filters_enabled = vec![false; self.columns.len()]; - self.visible_rows = (0..self.rows.len()).collect(); + self.search_rows = None; + self.filtered_rows = (0..self.rows.len()).collect(); + self.visible_rows = self.filtered_rows.clone(); // Stale — indexes into the old row set; `TablePanel` re-resolves right after. self.image_paths = vec![None; self.rows.len()]; self.values_generation += 1; @@ -756,6 +810,8 @@ impl QrateTableDelegate { /// What every row insert or delete invalidates. The view is rebuilt rather than patched: a /// filtered view's indices all move, and `recompute_visible` already drops the range for that. fn rows_changed(&mut self) { + // Hits are source rows, which an insert or delete just shifted. + self.search_rows = None; self.recompute_visible(); self.editing = EditState::Idle; self.values_generation += 1; @@ -1892,7 +1948,7 @@ mod app_tests { /// Visual results come best first, skip rows the index cannot rate, and stop at the limit. #[gpui::test] - fn ranked_matches_order_by_score_and_respect_the_limit(cx: &mut TestAppContext) { + fn search_rows_reorder_the_view_and_searches_still_see_every_row(cx: &mut TestAppContext) { let state = table(cx); cx.update(|cx| { state.update(cx, |state, _| { @@ -1908,13 +1964,30 @@ mod app_tests { Some("2.jpg") => Some(0.9), _ => None, }; - let rows: Vec = delegate - .ranked_matches(score, 5) - .into_iter() - .map(|(row, _)| row) - .collect(); - assert_eq!(rows, vec![2, 0]); - assert_eq!(delegate.ranked_matches(score, 1).len(), 1); + let rows = |d: &super::QrateTableDelegate| -> Vec { + d.ranked_rows(score) + .into_iter() + .map(|(_, row, _)| row) + .collect() + }; + assert_eq!(rows(delegate), vec![2, 0]); + + delegate.set_search_rows(Some(vec![2])); + assert_eq!(delegate.visible(), &[2]); + assert_eq!( + rows(delegate), + vec![2, 0], + "a narrowed view still ranks every row" + ); + + delegate.set_search_rows(Some(vec![3, 0])); + assert_eq!(delegate.visible(), &[3, 0], "hits keep their order"); + delegate.remove_rows(&[1]); + assert_eq!( + delegate.visible(), + &[0, 1, 2], + "a delete drops the stale hits" + ); }); }); } diff --git a/crates/table/src/panel.rs b/crates/table/src/panel.rs index f90caceb..0bfb66e9 100644 --- a/crates/table/src/panel.rs +++ b/crates/table/src/panel.rs @@ -7,6 +7,7 @@ use gpui_component::{ button::{Button, ButtonVariants as _}, h_flex, input::{Escape, Input, InputEvent, InputState, TextareaState}, + slider::{Slider, SliderEvent, SliderState}, table::{DataTable, TableEvent, TableState}, v_flex, }; @@ -100,6 +101,12 @@ pub struct TablePanel { /// The file a visual search with an empty query ranks against, from "Find similar items". similar: Option, _visual_sub: Subscription, + /// The last visual ranking as `(score, source_row, data_col)`, best first, so moving the + /// breadth slider re-cuts it without asking the model again. + ranking: Vec<(f32, usize, usize)>, + /// How far below the best visual hit still counts as a match. + breadth: Entity, + _breadth_sub: Subscription, replace_input: Entity, replace_open: bool, _replace_sub: Subscription, @@ -395,6 +402,17 @@ impl TablePanel { } }); + let breadth = cx.new(|_| { + SliderState::new() + .min(*visual::BREADTH_RANGE.start()) + .max(*visual::BREADTH_RANGE.end()) + .step(0.01) + .default_value(visual::BREADTH) + }); + let _breadth_sub = cx.subscribe(&breadth, |this, _, _: &SliderEvent, cx| { + this.show_ranking(cx) + }); + let _replace_sub = cx.subscribe(&replace_input, |this, _input, event: &InputEvent, cx| { if matches!(event, InputEvent::PressEnter { .. }) { this.replace(false, cx); @@ -414,16 +432,16 @@ impl TablePanel { search_open: false, search_matches: Vec::new(), search_ix: 0, - search_opts: SearchOpts { - files: true, - ..SearchOpts::default() - }, + search_opts: SearchOpts::default(), search_error: false, _search_sub, reading_documents: None, visual_query: None, similar: None, _visual_sub, + ranking: Vec::new(), + breadth, + _breadth_sub, replace_input, replace_open: false, _replace_sub, @@ -470,9 +488,19 @@ impl TablePanel { self.search_visual(cx); return; } + self.ranking.clear(); let needle = self.search_input.read(cx).value().to_string(); self.search_error = !needle.trim().is_empty() && compile_search(&needle, self.search_opts).is_none(); + // A search that reaches into linked files narrows the view to its hits. + let hits = (self.search_opts.files && !needle.trim().is_empty() && !self.search_error) + .then(|| { + self.state + .read(cx) + .delegate() + .hit_rows(&needle, self.search_opts) + }); + self.set_search_rows(hits, cx); self.search_matches = self .state .read(cx) @@ -499,8 +527,8 @@ impl TablePanel { (true, Some(path)) => (visual::Query::Like(path), std::time::Duration::ZERO), (true, None) => { self.visual_query = None; - self.search_matches.clear(); - cx.notify(); + self.ranking.clear(); + self.show_ranking(cx); return; } }; @@ -511,22 +539,51 @@ impl TablePanel { }; let score = score.await; this.update(cx, |this, cx| { - this.search_matches = score - .map(|score| { - this.state - .read(cx) - .delegate() - .ranked_matches(score, visual::RESULTS) - }) + // No scorer means the model is not loaded yet: show every row until it is. + this.ranking = score + .map(|score| this.state.read(cx).delegate().ranked_rows(score)) .unwrap_or_default(); - this.search_ix = 0; - this.select_current_match(cx); - cx.notify(); + this.show_ranking(cx); }) .ok(); })); } + /// Narrow the view to the visual ranking's hits within the breadth slider, best first, and step + /// through them in that order. An empty ranking shows every row. + fn show_ranking(&mut self, cx: &mut Context) { + let breadth = self.breadth.read(cx).value().start(); + let keep = visual::cutoff(self.ranking.iter().map(|hit| hit.0), breadth); + let hits = &self.ranking[..keep]; + let cols: std::collections::HashMap = + hits.iter().map(|&(_, row, col)| (row, col)).collect(); + self.set_search_rows( + (!hits.is_empty()).then(|| hits.iter().map(|hit| hit.1).collect()), + cx, + ); + self.search_matches = self + .state + .read(cx) + .delegate() + .visible() + .iter() + .enumerate() + .filter_map(|(view, source)| cols.get(source).map(|&col| (view, col))) + .collect(); + self.search_ix = 0; + self.select_current_match(cx); + cx.notify(); + } + + fn set_search_rows(&mut self, rows: Option>, cx: &mut Context) { + self.state.update(cx, |state, cx| { + if state.delegate_mut().set_search_rows(rows) { + cx.emit(TableChanged); + cx.notify(); + } + }); + } + /// Read the text of every linked document the search has not seen yet, off the UI thread, then /// search again so their hits join the results. fn read_documents(&mut self, cx: &mut Context) { @@ -639,16 +696,42 @@ impl TablePanel { .update(cx, |input, cx| input.focus(window, cx)); self.refresh_search(cx); } else { + self.ranking.clear(); + self.set_search_rows(None, cx); self.focus_handle.focus(window, cx); } cx.notify(); } + /// Open the find bar with its replace row. Replace has nothing to rewrite in a visual search. + pub fn open_replace(&mut self, window: &mut Window, cx: &mut Context) { + if !self.search_opts.visual { + self.replace_open = true; + } + if !self.search_open { + self.toggle_search(window, cx); + } + cx.notify(); + } + + /// Close the find bar if one of its own fields holds focus. `Escape` is bound in the `Input` + /// context, which the cell editor shares, so the caller must propagate when this declines. + pub fn escape_search(&mut self, window: &mut Window, cx: &mut Context) -> bool { + let focused = self.search_input.focus_handle(cx).is_focused(window) + || self.replace_input.focus_handle(cx).is_focused(window); + if focused { + self.dismiss_search(window, cx); + } + focused + } + fn dismiss_search(&mut self, window: &mut Window, cx: &mut Context) { if !self.search_open { return; } self.search_open = false; + self.ranking.clear(); + self.set_search_rows(None, cx); self.focus_handle.focus(window, cx); cx.notify(); } @@ -992,17 +1075,13 @@ impl TablePanel { (hooks.forget_suggestions)(cx); } } -} - -impl Focusable for TablePanel { - fn focus_handle(&self, _: &App) -> FocusHandle { - self.focus_handle.clone() - } -} -impl Render for TablePanel { - fn render(&mut self, window: &mut Window, cx: &mut Context) -> impl IntoElement { - let stripe = settings::effective_bool(crate::TABLE_STRIPES_KEY, cx); + /// The find bar, or nothing while it is closed. Drawn by whichever view is showing, so a search + /// carries on across Table and Gallery. + pub fn render_search_bar(&mut self, cx: &mut Context) -> AnyElement { + if !self.search_open { + return Empty.into_any_element(); + } let query = self.search_input.read(cx).value(); let visual_status = visual::status(cx); let count = if let Some(status) = self @@ -1031,6 +1110,192 @@ impl Render for TablePanel { SharedString::from("No results") }; let (border, muted) = (cx.theme().border, cx.theme().muted_foreground); + let opts = self.search_opts; + let toggle = |id: &'static str, + icon: Option, + label: &'static str, + tip: &'static str, + on: bool, + pick: fn(&mut SearchOpts) -> &mut bool| { + Button::new(id) + .ghost() + .small() + .selected(on) + // Text rules have no meaning for a query about what an image shows. + .disabled(opts.visual && id != "search-visual") + .tooltip(tip) + .map(|b| match icon { + Some(icon) => b.icon(icon), + None => b.label(label), + }) + .on_click(cx.listener(move |this, _, _, cx| this.toggle_opt(pick, cx))) + }; + let has_matches = !self.search_matches.is_empty(); + v_flex() + .flex_none() + .gap_1() + .px_2() + .py_2() + .border_b_1() + .border_color(border) + .child( + h_flex() + .gap_1() + .items_center() + .child( + Button::new("replace-toggle") + .ghost() + .small() + .icon(match self.replace_open { + true => IconName::ChevronDown, + false => IconName::ChevronRight, + }) + .disabled(opts.visual) + .tooltip("Toggle replace (Ctrl+H)") + .on_click(cx.listener(|this, _, _, cx| { + this.replace_open = !this.replace_open; + cx.notify(); + })), + ) + .child( + h_flex() + .flex_1() + .items_center() + .gap_1() + .child(div().flex_1().child(Input::new(&self.search_input))) + .child(toggle( + "search-case", + Some(IconName::CaseSensitive.into()), + "", + "Match case", + opts.case, + |o| &mut o.case, + )) + .child(toggle( + "search-word", + None, + "W", + "Match whole word", + opts.word, + |o| &mut o.word, + )) + .child(toggle( + "search-regex", + None, + ".*", + "Use regular expression", + opts.regex, + |o| &mut o.regex, + )) + .child(toggle( + "search-files", + Some(IconName::BookOpen.into()), + "", + "Include text inside linked files", + opts.files, + |o| &mut o.files, + )) + .child(toggle( + "search-visual", + Some(Icon::empty().path("icons/image.svg")), + "", + "Search by what images show", + opts.visual, + |o| &mut o.visual, + )), + ) + .child( + div() + .min_w(px(64.)) + .text_xs() + .text_color(muted) + .child(count), + ) + .when(opts.visual, |bar| match visual_status { + visual::Status::Missing | visual::Status::Failed(_) => bar.child( + Button::new("visual-install") + .small() + .label(visual::download_label()) + .on_click(|_, _, cx| visual::install(cx)), + ), + _ => bar.child( + div() + .id("visual-breadth") + .w(px(80.)) + .tooltip(|window, cx| { + gpui_component::tooltip::Tooltip::new("Fewer or more matches") + .build(window, cx) + }) + .child(Slider::new(&self.breadth)), + ), + }) + .child( + Button::new("search-prev") + .icon(IconName::ChevronUp) + .ghost() + .small() + .tooltip("Previous match (Shift+Enter)") + .on_click(cx.listener(|this, _, _, cx| this.goto_match(-1, cx))), + ) + .child( + Button::new("search-next") + .icon(IconName::ChevronDown) + .ghost() + .small() + .tooltip("Next match (Enter)") + .on_click(cx.listener(|this, _, _, cx| this.goto_match(1, cx))), + ) + .child( + Button::new("search-close") + .icon(IconName::Close) + .ghost() + .small() + .tooltip("Close find (Esc)") + .on_click( + cx.listener(|this, _, window, cx| this.dismiss_search(window, cx)), + ), + ), + ) + .when(self.replace_open && !opts.visual, |bar| { + bar.child( + h_flex() + .gap_1() + .items_center() + .pl_7() + .child(div().flex_1().child(Input::new(&self.replace_input))) + .child( + Button::new("replace-one") + .ghost() + .small() + .label("Replace") + .disabled(!has_matches) + .tooltip("Replace in this cell (Enter)") + .on_click(cx.listener(|this, _, _, cx| this.replace(false, cx))), + ) + .child( + Button::new("replace-all") + .ghost() + .small() + .label("All") + .disabled(!has_matches) + .tooltip("Replace every match in the visible rows") + .on_click(cx.listener(|this, _, _, cx| this.replace(true, cx))), + ), + ) + }) + .into_any_element() + } +} + +impl Focusable for TablePanel { + fn focus_handle(&self, _: &App) -> FocusHandle { + self.focus_handle.clone() + } +} + +impl Render for TablePanel { + fn render(&mut self, window: &mut Window, cx: &mut Context) -> impl IntoElement { + let stripe = settings::effective_bool(crate::TABLE_STRIPES_KEY, cx); v_flex() .size_full() @@ -1039,28 +1304,10 @@ impl Render for TablePanel { .id("table-panel") .role(Role::Group) .aria_label("Table") - .on_action(cx.listener(|this, _: &Search, window, cx| this.toggle_search(window, cx))) - .on_action(cx.listener(|this, _: &Replace, window, cx| { - this.replace_open = true; - if !this.search_open { - this.toggle_search(window, cx); - } - cx.notify(); - })) - // The find input propagates Escape (it doesn't consume it), so dismiss the bar here. - // - // Only when the bar's own fields hold the key, though. `Escape` is bound in the `Input` - // context, which the *cell* editor is in too, and this listener is an ancestor of both — - // so without the check, Escape in a cell editor either closed the find bar behind it or, - // with the bar already shut, was swallowed by a no-op handler and did nothing at all. - // An action stops propagating by default, so declining it has to be explicit. + // Search, Replace and the find bar's own Escape are handled by `ViewsPanel`, which draws + // the bar above every view. An action stops propagating by default, so declining an + // Escape that is not ours has to be explicit. .on_action(cx.listener(|this, _: &Escape, window, cx| { - if this.search_input.focus_handle(cx).is_focused(window) - || this.replace_input.focus_handle(cx).is_focused(window) - { - this.dismiss_search(window, cx); - return; - } // Like a cell edit, a note owns the blur that Escape is about to cause. Drop that // ownership first or `_note_sub` will mistake the blur for a commit. if this.cancel_note_edit(window, cx) { @@ -1128,189 +1375,6 @@ impl Render for TablePanel { ) .p_2() .gap_2() - // Find bar renders here, not in `title_suffix`: the parent `TabPanel` never observes us to redraw it. - .when(self.search_open, |this| { - let opts = self.search_opts; - let toggle = |id: &'static str, - icon: Option, - label: &'static str, - tip: &'static str, - on: bool, - pick: fn(&mut SearchOpts) -> &mut bool| { - Button::new(id) - .ghost() - .small() - .selected(on) - // Text rules have no meaning for a query about what an image shows. - .disabled(opts.visual && id != "search-visual") - .tooltip(tip) - .map(|b| match icon { - Some(icon) => b.icon(icon), - None => b.label(label), - }) - .on_click(cx.listener(move |this, _, _, cx| this.toggle_opt(pick, cx))) - }; - let has_matches = !self.search_matches.is_empty(); - this.child( - v_flex() - .flex_none() - .gap_1() - .pb_2() - .border_b_1() - .border_color(border) - .child( - h_flex() - .gap_1() - .items_center() - .child( - Button::new("replace-toggle") - .ghost() - .small() - .icon(match self.replace_open { - true => IconName::ChevronDown, - false => IconName::ChevronRight, - }) - .tooltip("Toggle replace (Ctrl+H)") - .on_click(cx.listener(|this, _, _, cx| { - this.replace_open = !this.replace_open; - cx.notify(); - })), - ) - .child( - h_flex() - .flex_1() - .items_center() - .gap_1() - .child(div().flex_1().child(Input::new(&self.search_input))) - .child(toggle( - "search-case", - Some(IconName::CaseSensitive.into()), - "", - "Match case", - opts.case, - |o| &mut o.case, - )) - .child(toggle( - "search-word", - None, - "W", - "Match whole word", - opts.word, - |o| &mut o.word, - )) - .child(toggle( - "search-regex", - None, - ".*", - "Use regular expression", - opts.regex, - |o| &mut o.regex, - )) - .child(toggle( - "search-files", - Some(IconName::BookOpen.into()), - "", - "Include text inside linked files", - opts.files, - |o| &mut o.files, - )) - .child(toggle( - "search-visual", - Some(Icon::empty().path("icons/image.svg")), - "", - "Search by what images show", - opts.visual, - |o| &mut o.visual, - )), - ) - .child( - div() - .min_w(px(64.)) - .text_xs() - .text_color(muted) - .child(count), - ) - .when( - opts.visual - && matches!( - visual_status, - visual::Status::Missing | visual::Status::Failed(_) - ), - |bar| { - bar.child( - Button::new("visual-install") - .small() - .label(visual::download_label()) - .on_click(|_, _, cx| visual::install(cx)), - ) - }, - ) - .child( - Button::new("search-prev") - .icon(IconName::ChevronUp) - .ghost() - .small() - .tooltip("Previous match (Shift+Enter)") - .on_click( - cx.listener(|this, _, _, cx| this.goto_match(-1, cx)), - ), - ) - .child( - Button::new("search-next") - .icon(IconName::ChevronDown) - .ghost() - .small() - .tooltip("Next match (Enter)") - .on_click( - cx.listener(|this, _, _, cx| this.goto_match(1, cx)), - ), - ) - .child( - Button::new("search-close") - .icon(IconName::Close) - .ghost() - .small() - .tooltip("Close find (Esc)") - .on_click(cx.listener(|this, _, window, cx| { - this.dismiss_search(window, cx) - })), - ), - ) - .when(self.replace_open, |bar| { - bar.child( - h_flex() - .gap_1() - .items_center() - .pl_7() - .child(div().flex_1().child(Input::new(&self.replace_input))) - .child( - Button::new("replace-one") - .ghost() - .small() - .label("Replace") - .disabled(!has_matches) - .tooltip("Replace in this cell (Enter)") - .on_click(cx.listener(|this, _, _, cx| { - this.replace(false, cx) - })), - ) - .child( - Button::new("replace-all") - .ghost() - .small() - .label("All") - .disabled(!has_matches) - .tooltip("Replace every match in the visible rows") - .on_click( - cx.listener(|this, _, _, cx| { - this.replace(true, cx) - }), - ), - ), - ) - }), - ) - }) .child( div() .flex_1() diff --git a/crates/table/src/visual.rs b/crates/table/src/visual.rs index dd411620..badf5381 100644 --- a/crates/table/src/visual.rs +++ b/crates/table/src/visual.rs @@ -18,11 +18,25 @@ const BATCH: usize = 16; /// Batches between index saves, about a minute of CPU work, so quitting mid-index keeps progress. const SAVE_EVERY: usize = 40; -/// How many rows a visual search returns, best first. +/// The most rows a visual search keeps, however broad the search. +const MAX_RESULTS: usize = 500; + +/// The breadth a new search starts at, and the range its slider allows. +pub(crate) const BREADTH: f32 = 0.15; +pub(crate) const BREADTH_RANGE: std::ops::RangeInclusive = 0.02..=0.4; + +/// How many of `scores` (best first) to keep: those within `breadth` of the best, proportionally. /// -/// ponytail: a fixed count, not a similarity threshold. CLIP scores cluster tightly, so a cut-off -/// needs tuning against real collections first. -pub(crate) const RESULTS: usize = 50; +/// ponytail: a proportional gap from the best hit, so text queries (best around 0.3) and find +/// similar (best 1.0) share one slider. Tuned on one collection; calibrate per model if it misjudges. +pub(crate) fn cutoff(scores: impl IntoIterator, breadth: f32) -> usize { + let mut scores = scores.into_iter(); + let Some(best) = scores.next() else { + return 0; + }; + let floor = best * (1.0 - breadth); + (1 + scores.take_while(|&score| score >= floor).count()).min(MAX_RESULTS) +} #[derive(Clone, PartialEq)] pub(crate) enum Status { @@ -316,3 +330,25 @@ pub(crate) fn scorer( pub(crate) fn find_similar(path: PathBuf, cx: &mut App) { state(cx).similar = Some(path); } + +#[cfg(test)] +mod tests { + use super::cutoff; + + #[test] + fn cutoff_keeps_hits_near_the_best_one() { + assert_eq!(cutoff([], 0.15), 0); + assert_eq!(cutoff([0.30, 0.28, 0.26, 0.20], 0.15), 3); + assert_eq!( + cutoff([0.30, 0.28, 0.26, 0.20], 0.02), + 1, + "the best hit always stays" + ); + assert_eq!( + cutoff([1.0, 0.9, 0.5], 0.15), + 2, + "find similar starts from itself" + ); + assert_eq!(cutoff(vec![0.3; 900], 0.4), 500, "capped"); + } +} diff --git a/crates/workspace/src/views/gallery.rs b/crates/workspace/src/views/gallery.rs index 851f08e8..574fdfb4 100644 --- a/crates/workspace/src/views/gallery.rs +++ b/crates/workspace/src/views/gallery.rs @@ -27,6 +27,7 @@ pub(super) fn render( state: Option>>, width: Pixels, cols: usize, + scroll: &UniformListScrollHandle, focus: &FocusHandle, cx: &mut App, ) -> AnyElement { @@ -65,6 +66,7 @@ pub(super) fn render( .collect() }, ) + .track_scroll(scroll) .size_full() .p_2() .into_any_element() diff --git a/crates/workspace/src/views/mod.rs b/crates/workspace/src/views/mod.rs index ea37c618..05d67bce 100644 --- a/crates/workspace/src/views/mod.rs +++ b/crates/workspace/src/views/mod.rs @@ -19,11 +19,12 @@ use gpui_component::{ button::Button, dock::{BasePanel, DockArea, Panel, PanelEvent}, h_flex, + input::Escape, slider::{Slider, SliderEvent, SliderState}, tab::{Tab, TabBar}, v_flex, }; -use table::{QrateTableDelegate, TableChanged, TablePanel, TableStateHandle}; +use table::{QrateTableDelegate, Replace, Search, TableChanged, TablePanel, TableStateHandle}; use crate::ViewerScope; @@ -136,6 +137,12 @@ pub struct ViewsPanel { thumb: Entity, /// Persists the count as it is dragged, and repaints the cards to match. _thumb_sub: Subscription, + /// Repaints the find bar, which this panel draws on the table's behalf, as the search moves. + _table_sub: Subscription, + gallery_scroll: UniformListScrollHandle, + /// The row the gallery last scrolled to, so it follows the selection (stepping through search + /// hits) without pulling back a card the user has scrolled away from. + gallery_followed: Option, } impl ViewsPanel { @@ -169,7 +176,7 @@ impl ViewsPanel { }); let mut this = Self { focus_handle: cx.focus_handle(), - table, + table: table.clone(), dock_area, view: ViewMode::parse(&settings::effective_text(VIEW_MODE_KEY, cx)), body_width: px(0.), @@ -182,6 +189,9 @@ impl ViewsPanel { _viewer_sub: cx.observe_global::(|_, cx| cx.notify()), thumb, _thumb_sub, + _table_sub: cx.observe(&table, |_, _, cx| cx.notify()), + gallery_scroll: UniformListScrollHandle::new(), + gallery_followed: None, }; this.bind(cx); cx.set_global(ActiveViewsPanel(cx.entity().downgrade())); @@ -337,11 +347,8 @@ impl Panel for ViewsPanel { } /// Rendered by `TabPanel::render_toolbar` immediately left of the ⋯ menu, forced to - /// `.xsmall().ghost()` by the library. Only the grid has anything to find in. + /// `.xsmall().ghost()` by the library. Every view shows the same rows, so every view can find in them. fn toolbar_buttons(&mut self, _w: &mut Window, _cx: &mut Context) -> Option> { - if self.view != ViewMode::Table { - return None; - } let table = self.table.downgrade(); Some(vec![ Button::new("table-search") @@ -366,6 +373,27 @@ impl Render for ViewsPanel { .id("views-panel") .role(Role::Group) .aria_label("Views") + // The find bar lives above whichever view is showing, so its keys are handled here too. + .on_action(cx.listener(|this, _: &Search, window, cx| { + this.table + .update(cx, |table, cx| table.toggle_search(window, cx)) + })) + .on_action(cx.listener(|this, _: &Replace, window, cx| { + this.table + .update(cx, |table, cx| table.open_replace(window, cx)) + })) + .on_action(cx.listener(|this, _: &Escape, window, cx| { + if !this + .table + .update(cx, |table, cx| table.escape_search(window, cx)) + { + cx.propagate(); + } + })) + .child( + self.table + .update(cx, |table, cx| table.render_search_bar(cx)), + ) .child( div() .flex_1() @@ -391,13 +419,35 @@ impl Render for ViewsPanel { // chained-builder types meeting in one `match` overflows rustc's stack. .child(match self.view { ViewMode::Table => self.table.clone().into_any_element(), - ViewMode::Gallery => gallery::render( - self.state.as_ref().and_then(WeakEntity::upgrade), - self.body_width, - self.thumb.read(cx).value().start().round() as usize, - &self.focus_handle, - cx, - ), + ViewMode::Gallery => { + let state = self.state.as_ref().and_then(WeakEntity::upgrade); + let cols = self.thumb.read(cx).value().start().round() as usize; + let cursor = state.as_ref().and_then(|state| { + let delegate = state.read(cx).delegate(); + match delegate.selection()? { + table::Selection::Cell { row, .. } + | table::Selection::Row(row) => delegate.view_row(row), + table::Selection::Column(_) => None, + } + }); + if cursor != self.gallery_followed { + self.gallery_followed = cursor; + if let Some(view) = cursor { + self.gallery_scroll.scroll_to_item( + view / cols.max(1), + ScrollStrategy::Nearest, + ); + } + } + gallery::render( + state, + self.body_width, + cols, + &self.gallery_scroll, + &self.focus_handle, + cx, + ) + } }) .children(crate::viewer::viewer_in(ViewerScope::Centre, cx)), ) From 6447b639d2268392808674cf277f7c42a3326928 Mon Sep 17 00:00:00 2001 From: devnull03 <56480041+devnull03@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:55:45 -0700 Subject: [PATCH 06/12] feat(table): store visual search vectors in the project file Vectors now live in a __visual_index table inside the .qrate, created on first write, with the model that built them recorded in __settings. Paths are kept relative to the files folder and staleness is judged on file length, so a project copied with its files keeps its index. Each batch is written as it is embedded, outside the index lock. Replaces the global visual-index.bin. --- CLAUDE.md | 2 +- crates/settings/src/project.rs | 108 +++++++++++++++++ crates/table/src/visual.rs | 175 ++++++++++++++++++--------- crates/visual-search/src/index.rs | 189 ------------------------------ crates/visual-search/src/lib.rs | 7 +- 5 files changed, 231 insertions(+), 250 deletions(-) delete mode 100644 crates/visual-search/src/index.rs diff --git a/CLAUDE.md b/CLAUDE.md index 0d4d7b55..ede29088 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -83,7 +83,7 @@ problem — read the `--check` diff and apply it by hand rather than fighting th | `plugin-host` | the Luau runtime that loads and runs plugins | | `plugin-api` | the types a plugin sees — see the three-repo rule below | | `preview` | turns a linked file into pixels — the format ladder and the thumbnail cache. Native decoders (PDF, video, RAW) belong here so they never reach `table` | -| `visual-search` | CLIP on the CPU through candle (pure Rust), its pinned weights download, and the on-disk image index. No gpui — `table::visual` owns the job, the search bar, and "Find similar items". Weights live in `/models`, never in the install | +| `visual-search` | CLIP on the CPU through candle (pure Rust) and its pinned weights download. No gpui — `table::visual` owns the index job, the search bar, and "Find similar items", and stores the vectors in the project's `.qrate` (`__visual_index`). Weights live in `/models`, never in the install | | `ai` | two halves: `agent.rs` is the *shipped* external-agent contract (see `AGENTS.md`); the rest is traits + Cohere/mock providers for planned AI review/embedding, deliberately unfinished | ## Project Status Tracking diff --git a/crates/settings/src/project.rs b/crates/settings/src/project.rs index abbbd1af..dfa8c1b1 100644 --- a/crates/settings/src/project.rs +++ b/crates/settings/src/project.rs @@ -671,6 +671,94 @@ pub fn write_notes(path: &Path, source: &str, notes: &[StoredNote]) -> Result<() Ok(()) } +/// Created on first write, like `__notes`, so opening a project stays read-only. Keyed by the path +/// as the caller stores it (relative to the files folder where it can be). +const VISUAL_INDEX_DDL: &str = r#" + CREATE TABLE IF NOT EXISTS __visual_index ( + path TEXT PRIMARY KEY, + len INTEGER NOT NULL, + vector BLOB NOT NULL + ); +"#; + +/// `__settings` key naming the model that built `__visual_index`. Vectors from different models +/// cannot be compared, so a mismatch reads as an empty index and the next write clears it. +const VISUAL_MODEL_KEY: &str = "visual_model"; + +/// One image's vector: the stored path, the file's length when embedded, and the vector. +pub type VisualEntry = (String, u64, Vec); + +/// Every vector `model` stored in this project. Empty when there is no index yet, or another model +/// built it. +pub fn read_visual_index(path: &Path, model: &str) -> Result> { + let conn = open_ro(path)?; + let exists: i64 = conn.query_row( + "SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name = '__visual_index'", + [], + |r| r.get(0), + )?; + let built_by: Option = conn + .query_row( + "SELECT value FROM __settings WHERE key = ?1", + params![VISUAL_MODEL_KEY], + |r| r.get(0), + ) + .optional()?; + if exists == 0 || built_by.as_deref() != Some(model) { + return Ok(Vec::new()); + } + let mut stmt = conn.prepare("SELECT path, len, vector FROM __visual_index")?; + let rows = stmt.query_map([], |r| { + let bytes: Vec = r.get(2)?; + let vector = bytes + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect(); + Ok((r.get(0)?, r.get::<_, i64>(1)? as u64, vector)) + })?; + rows.collect::>() + .context("Read visual index") +} + +/// Store `entries`, replacing any with the same path, in one transaction. Clears what another model +/// stored first. +pub fn write_visual_index(path: &Path, model: &str, entries: &[VisualEntry]) -> Result<()> { + let mut conn = open_rw(path)?; + let tx = conn.transaction()?; + tx.execute_batch(VISUAL_INDEX_DDL) + .context("Create __visual_index")?; + let built_by: Option = tx + .query_row( + "SELECT value FROM __settings WHERE key = ?1", + params![VISUAL_MODEL_KEY], + |r| r.get(0), + ) + .optional()?; + if built_by.as_deref() != Some(model) { + tx.execute("DELETE FROM __visual_index", []) + .context("Clear another model's vectors")?; + tx.execute( + "INSERT INTO __settings(key, value) VALUES (?1, ?2) + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + params![VISUAL_MODEL_KEY, model], + ) + .context("Record visual model")?; + } + { + let mut stmt = tx.prepare( + "INSERT INTO __visual_index(path, len, vector) VALUES (?1, ?2, ?3) + ON CONFLICT(path) DO UPDATE SET len = excluded.len, vector = excluded.vector", + )?; + for (file, len, vector) in entries { + let bytes: Vec = vector.iter().flat_map(|v| v.to_le_bytes()).collect(); + stmt.execute(params![file, *len as i64, bytes]) + .context("Insert vector")?; + } + } + tx.commit().context("Commit visual index")?; + Ok(()) +} + /// Debounced background writer for `__settings` values. The dock-layout and /// window-bounds observers fire on every drag event; latest value per /// (file, key) wins, and one thread serves all project files — no lifecycle @@ -927,6 +1015,26 @@ mod tests { .unwrap(); } + #[test] + fn visual_index_round_trips_and_forgets_another_models_vectors() { + let path = tempfile("visual.qrate"); + blank_project(&path); + assert!(read_visual_index(&path, "clip-a").unwrap().is_empty()); + + write_visual_index(&path, "clip-a", &[("a.jpg".into(), 6, vec![0.6, 0.8])]).unwrap(); + write_visual_index(&path, "clip-a", &[("a.jpg".into(), 7, vec![1.0, 0.0])]).unwrap(); + assert_eq!( + read_visual_index(&path, "clip-a").unwrap(), + vec![("a.jpg".to_string(), 7, vec![1.0, 0.0])], + "a re-embedded file replaces its row" + ); + assert!(read_visual_index(&path, "clip-b").unwrap().is_empty()); + + write_visual_index(&path, "clip-b", &[("b.jpg".into(), 1, vec![0.0])]).unwrap(); + assert_eq!(read_visual_index(&path, "clip-b").unwrap().len(), 1); + assert!(read_visual_index(&path, "clip-a").unwrap().is_empty()); + } + /// A project written before notes carried a date and an author must still open, still read its /// notes, and gain the columns on the next save — losing a catalogue's marginalia to a schema /// change is the one outcome here that cannot be undone. diff --git a/crates/table/src/visual.rs b/crates/table/src/visual.rs index badf5381..cb99ca85 100644 --- a/crates/table/src/visual.rs +++ b/crates/table/src/visual.rs @@ -3,21 +3,19 @@ //! //! Shared across windows as a global, because the model holds 600 MB and one copy is plenty. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; use gpui::{App, Global, SharedString, Task}; -use visual_search::{Clip, Index, Stamp}; +use settings::project::{CurrentProject, FILES_FOLDER_KEY, VisualEntry}; +use visual_search::Clip; /// Files embedded per model call. Larger batches barely help on a CPU and delay the progress count. const BATCH: usize = 16; -/// Batches between index saves, about a minute of CPU work, so quitting mid-index keeps progress. -const SAVE_EVERY: usize = 40; - /// The most rows a visual search keeps, however broad the search. const MAX_RESULTS: usize = 500; @@ -47,10 +45,18 @@ pub(crate) enum Status { Failed(SharedString), } +/// Each linked file's vector, with the file length it was embedded at. +/// +/// ponytail: staleness is judged on length alone, so a copied project keeps its vectors when file +/// times change. Store a content hash if same-length edits turn out to matter. +type Index = HashMap)>; + pub(crate) struct Visual { pub status: Status, clip: Option>, index: Arc>, + /// The `.qrate` file `index` belongs to. Another project starts from its own stored index. + project: Option, /// Files the indexing job has already been handed this session, so asking again is free. seen: HashSet, job: Option>, @@ -68,10 +74,6 @@ fn model_dir() -> Option { ) } -fn index_file() -> Option { - Some(settings::data_dir()?.join("visual-index.bin")) -} - pub(crate) fn init(cx: &mut App) { if cx.has_global::() { return; @@ -84,7 +86,8 @@ pub(crate) fn init(cx: &mut App) { Status::Missing }, clip: None, - index: Arc::new(Mutex::new(Index::new(visual_search::MODEL))), + index: Arc::default(), + project: None, seen: HashSet::new(), job: None, similar: None, @@ -170,50 +173,72 @@ pub(crate) fn install(cx: &mut App) { state(cx).job = Some(job); } -/// Embed any of `paths` the index does not already hold for their current contents. Returns at -/// once; progress shows in [`Status::Indexing`]. +/// Embed any of `paths` the index does not already hold at their current length, storing the +/// vectors in the open project. Returns at once; progress shows in [`Status::Indexing`]. pub(crate) fn index(paths: Vec, cx: &mut App) { + let project = cx.try_global::().map(|project| { + let folder = project + .data + .values + .get(FILES_FOLDER_KEY) + .map(|v| PathBuf::from(v.text().to_string())) + .unwrap_or_default(); + (project.file.clone(), folder) + }); let visual = cx.global::(); + let Some(dir) = model_dir() else { + return; + }; if visual.status != Status::Ready || visual.job.is_some() { return; } + let switched = visual.project != project.as_ref().map(|(file, _)| file.clone()); let fresh: Vec = paths .into_iter() - .filter(|path| !visual.seen.contains(path)) + .filter(|path| switched || !visual.seen.contains(path)) .collect(); - let (Some(dir), Some(file)) = (model_dir(), index_file()) else { - return; - }; if fresh.is_empty() { return; } - let (index, clip) = (visual.index.clone(), visual.clip.clone()); - state(cx).seen.extend(fresh.iter().cloned()); + let clip = visual.clip.clone(); + let visual = state(cx); + if switched { + visual.project = project.as_ref().map(|(file, _)| file.clone()); + visual.seen.clear(); + visual.index = Arc::default(); + } + visual.seen.extend(fresh.iter().cloned()); + let index = visual.index.clone(); let job = cx.spawn(async move |cx| { let prepared = cx .background_executor() .spawn({ - let index = index.clone(); - let file = file.clone(); + let (index, project) = (index.clone(), project.clone()); async move { let clip = match clip { Some(clip) => clip, None => Arc::new(Clip::load(&dir).map_err(|err| format!("{err:#}"))?), }; - // Stat outside the lock: searches rank rows on the UI thread through this index. - let stamped: Vec<(PathBuf, Stamp)> = fresh - .into_iter() - .filter_map(|path| Stamp::of(&path).map(|stamp| (path, stamp))) - .collect(); - let empty = lock(&index).is_empty(); - let loaded = empty.then(|| Index::load(&file, visual_search::MODEL)); - let mut index = lock(&index); - if let Some(loaded) = loaded { - *index = loaded; + if switched && let Some((file, folder)) = &project { + match settings::project::read_visual_index(file, visual_search::MODEL) { + Ok(stored) => lock(&index).extend( + stored + .into_iter() + .map(|(path, len, vector)| (folder.join(path), (len, vector))), + ), + Err(err) => log::warn!( + "could not read the project's visual search index, rebuilding it: {err:#}" + ), + } } - let stale: Vec<(PathBuf, Stamp)> = stamped + // Stat outside the lock: searches rank rows on the UI thread through this index. + let stale: Vec<(PathBuf, u64)> = fresh .into_iter() - .filter(|(path, stamp)| !index.is_current(path, *stamp)) + .filter_map(|path| { + let len = std::fs::metadata(&path).ok()?.len(); + let kept = lock(&index).get(&path).map(|(kept, _)| *kept); + (kept != Some(len)).then_some((path, len)) + }) .collect(); Ok::<_, String>((clip, stale)) } @@ -241,15 +266,27 @@ pub(crate) fn index(paths: Vec, cx: &mut App) { total, } }); - let (clip, index, file, batch) = - (clip.clone(), index.clone(), file.clone(), batch.to_vec()); - let last = (at + 1) * BATCH >= total; + let (clip, index, project, batch) = + (clip.clone(), index.clone(), project.clone(), batch.to_vec()); cx.background_executor() .spawn(async move { - embed_batch(&clip, &index, batch); - if last || at % SAVE_EVERY == SAVE_EVERY - 1 { - save(&index, &file); + let embedded = embed_batch(&clip, batch); + if let Some((file, folder)) = &project { + let stored: Vec = embedded + .iter() + .map(|(path, len, vector)| (stored_path(path, folder), *len, vector.clone())) + .collect(); + if let Err(err) = + settings::project::write_visual_index(file, visual_search::MODEL, &stored) + { + log::warn!("could not save visual search vectors to the project, they will be rebuilt: {err:#}"); + } } + lock(&index).extend( + embedded + .into_iter() + .map(|(path, len, vector)| (path, (len, vector))), + ); }) .await; } @@ -269,31 +306,40 @@ fn lock(index: &Mutex) -> std::sync::MutexGuard<'_, Index> { index.lock().unwrap_or_else(|err| err.into_inner()) } -fn save(index: &Mutex, file: &Path) { - if let Err(err) = lock(index).save(file) { - log::warn!("could not save the visual search index, it will be rebuilt next launch: {err}"); +/// `path` relative to the files folder with `/` separators, so a project moved with its files, or +/// to another platform, still finds its vectors. Absolute when the file lives elsewhere. +fn stored_path(path: &Path, folder: &Path) -> String { + match path.strip_prefix(folder) { + Ok(relative) if !folder.as_os_str().is_empty() => relative + .components() + .map(|part| part.as_os_str().to_string_lossy()) + .collect::>() + .join("/"), + _ => path.to_string_lossy().into_owned(), } } -fn embed_batch(clip: &Clip, index: &Mutex, batch: Vec<(PathBuf, Stamp)>) { +fn embed_batch(clip: &Clip, batch: Vec<(PathBuf, u64)>) -> Vec<(PathBuf, u64, Vec)> { let (files, images): (Vec<_>, Vec<_>) = batch .into_iter() - .filter_map(|(path, stamp)| { + .filter_map(|(path, len)| { let pixels = preview::thumbnail_pixels(&path, preview::CARD, 0)?; - Some(((path, stamp), pixels)) + Some(((path, len), pixels)) }) .unzip(); if images.is_empty() { - return; + return Vec::new(); } match clip.embed_images(&images) { - Ok(vectors) => { - let mut index = lock(index); - for ((path, stamp), vector) in files.into_iter().zip(vectors) { - index.insert(path, stamp, vector); - } + Ok(vectors) => files + .into_iter() + .zip(vectors) + .map(|((path, len), vector)| (path, len, vector)) + .collect(), + Err(err) => { + log::warn!("visual search skipped {} files: {err:#}", files.len()); + Vec::new() } - Err(err) => log::warn!("visual search skipped {} files: {err:#}", files.len()), } } @@ -303,7 +349,7 @@ pub(crate) enum Query { Like(PathBuf), } -/// A scoring function over files for [`crate::delegate::QrateTableDelegate::ranked_matches`], or +/// A scoring function over files for [`crate::delegate::QrateTableDelegate::ranked_rows`], or /// `None` when the model is not loaded or the query cannot be embedded. pub(crate) fn scorer( query: Query, @@ -317,11 +363,11 @@ pub(crate) fn scorer( .embed_text(&text) .map_err(|err| log::warn!("could not embed a visual search query: {err:#}")) .ok()?, - Query::Like(path) => lock(&index).vector(&path)?.to_vec(), + Query::Like(path) => lock(&index).get(&path)?.1.clone(), }; Some(move |path: &Path| { let index = lock(&index); - Some(visual_search::similarity(&vector, index.vector(path)?)) + Some(visual_search::similarity(&vector, &index.get(path)?.1)) }) }) } @@ -333,7 +379,26 @@ pub(crate) fn find_similar(path: PathBuf, cx: &mut App) { #[cfg(test)] mod tests { - use super::cutoff; + use std::path::Path; + + use super::{cutoff, stored_path}; + + #[test] + fn stored_paths_are_relative_to_the_files_folder() { + let folder = Path::new("/archive/files"); + assert_eq!( + stored_path(&folder.join("box 1").join("a.jpg"), folder), + "box 1/a.jpg" + ); + assert_eq!( + stored_path(Path::new("/elsewhere/b.jpg"), folder), + "/elsewhere/b.jpg" + ); + assert_eq!( + stored_path(Path::new("/x/c.jpg"), Path::new("")), + "/x/c.jpg" + ); + } #[test] fn cutoff_keeps_hits_near_the_best_one() { diff --git a/crates/visual-search/src/index.rs b/crates/visual-search/src/index.rs deleted file mode 100644 index 3a8fa223..00000000 --- a/crates/visual-search/src/index.rs +++ /dev/null @@ -1,189 +0,0 @@ -//! Every embedded file's vector, kept on disk between launches. -//! -//! Keyed by path and stamped with the file's size and modification time, so an edited or replaced -//! file is embedded again and an untouched one never is. -//! -//! ponytail: entries for deleted files are never dropped, at 2 KB each. Prune on save if a -//! long-lived index grows noticeably. - -use std::collections::HashMap; -use std::fs; -use std::io::{self, BufReader, BufWriter, Read, Write}; -use std::path::{Path, PathBuf}; -use std::time::{Duration, UNIX_EPOCH}; - -const MAGIC: &[u8; 8] = b"qrvidx01"; - -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub struct Stamp { - len: u64, - modified: Duration, -} - -impl Stamp { - pub fn of(path: &Path) -> Option { - let meta = fs::metadata(path).ok()?; - Some(Self { - len: meta.len(), - modified: meta.modified().ok()?.duration_since(UNIX_EPOCH).ok()?, - }) - } -} - -#[derive(Default)] -pub struct Index { - model: String, - entries: HashMap)>, -} - -impl Index { - pub fn new(model: &str) -> Self { - Self { - model: model.to_owned(), - entries: HashMap::new(), - } - } - - /// The index saved at `file`, or an empty one when there is none, it is unreadable, or it was - /// built by a different model. - pub fn load(file: &Path, model: &str) -> Self { - match read(file) { - Ok(index) if index.model == model => index, - Ok(_) => { - log::info!("the visual search index was built by another model, rebuilding it"); - Self::new(model) - } - Err(err) if err.kind() == io::ErrorKind::NotFound => Self::new(model), - Err(err) => { - log::warn!("could not read the visual search index, rebuilding it: {err}"); - Self::new(model) - } - } - } - - /// Written beside `file` and renamed over it, so a crash mid-write leaves the previous index. - pub fn save(&self, file: &Path) -> io::Result<()> { - let partial = file.with_extension("part"); - let mut out = BufWriter::new(fs::File::create(&partial)?); - out.write_all(MAGIC)?; - write_bytes(&mut out, self.model.as_bytes())?; - for (path, (stamp, vector)) in &self.entries { - let Some(path) = path.to_str() else { - continue; - }; - write_bytes(&mut out, path.as_bytes())?; - out.write_all(&stamp.len.to_le_bytes())?; - out.write_all(&stamp.modified.as_secs().to_le_bytes())?; - out.write_all(&stamp.modified.subsec_nanos().to_le_bytes())?; - out.write_all(&(vector.len() as u32).to_le_bytes())?; - for value in vector { - out.write_all(&value.to_le_bytes())?; - } - } - out.into_inner() - .map_err(|err| err.into_error())? - .sync_all()?; - fs::rename(partial, file) - } - - pub fn is_current(&self, path: &Path, stamp: Stamp) -> bool { - self.entries - .get(path) - .is_some_and(|(kept, _)| *kept == stamp) - } - - pub fn insert(&mut self, path: PathBuf, stamp: Stamp, vector: Vec) { - self.entries.insert(path, (stamp, vector)); - } - - pub fn is_empty(&self) -> bool { - self.entries.is_empty() - } - - pub fn vector(&self, path: &Path) -> Option<&[f32]> { - self.entries.get(path).map(|(_, vector)| vector.as_slice()) - } -} - -fn read(file: &Path) -> io::Result { - let mut input = BufReader::new(fs::File::open(file)?); - let mut magic = [0; 8]; - input.read_exact(&mut magic)?; - if &magic != MAGIC { - return Err(io::Error::new(io::ErrorKind::InvalidData, "not an index")); - } - let model = String::from_utf8(read_bytes(&mut input)?) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; - let mut index = Index::new(&model); - loop { - let path = match read_bytes(&mut input) { - Ok(path) => path, - Err(err) if err.kind() == io::ErrorKind::UnexpectedEof => return Ok(index), - Err(err) => return Err(err), - }; - let path = String::from_utf8(path) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; - let len = u64::from_le_bytes(read_array(&mut input)?); - let secs = u64::from_le_bytes(read_array(&mut input)?); - let nanos = u32::from_le_bytes(read_array(&mut input)?); - let dim = u32::from_le_bytes(read_array(&mut input)?) as usize; - let vector = (0..dim) - .map(|_| read_array(&mut input).map(f32::from_le_bytes)) - .collect::>()?; - let stamp = Stamp { - len, - modified: Duration::new(secs, nanos), - }; - index.insert(path.into(), stamp, vector); - } -} - -fn write_bytes(out: &mut impl Write, bytes: &[u8]) -> io::Result<()> { - out.write_all(&(bytes.len() as u32).to_le_bytes())?; - out.write_all(bytes) -} - -fn read_bytes(input: &mut impl Read) -> io::Result> { - let len = u32::from_le_bytes(read_array(input)?) as usize; - let mut bytes = vec![0; len]; - input.read_exact(&mut bytes)?; - Ok(bytes) -} - -fn read_array(input: &mut impl Read) -> io::Result<[u8; N]> { - let mut bytes = [0; N]; - input.read_exact(&mut bytes)?; - Ok(bytes) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn round_trips_and_forgets_another_models_vectors() { - let dir = std::env::temp_dir().join("qrate-visual-index-test"); - fs::create_dir_all(&dir).unwrap(); - let file = dir.join("index.bin"); - let photo = dir.join("photo.jpg"); - fs::write(&photo, b"pixels").unwrap(); - let stamp = Stamp::of(&photo).unwrap(); - - let mut index = Index::new("clip-a"); - index.insert(photo.clone(), stamp, vec![0.6, 0.8]); - index.save(&file).unwrap(); - - let loaded = Index::load(&file, "clip-a"); - assert!(loaded.is_current(&photo, stamp)); - assert_eq!(loaded.vector(&photo), Some(&[0.6, 0.8][..])); - - fs::write(&photo, b"a different, longer set of pixels").unwrap(); - assert!( - !loaded.is_current(&photo, Stamp::of(&photo).unwrap()), - "an edited file is stale" - ); - assert!(Index::load(&file, "clip-b").vector(&photo).is_none()); - - let _ = fs::remove_dir_all(&dir); - } -} diff --git a/crates/visual-search/src/lib.rs b/crates/visual-search/src/lib.rs index dfb52d53..c6975a61 100644 --- a/crates/visual-search/src/lib.rs +++ b/crates/visual-search/src/lib.rs @@ -1,14 +1,13 @@ //! Searching a collection by what its images show. //! //! CLIP turns an image and a sentence into vectors in the same space, so "a sawmill by the water" -//! lands near photographs of one. Every linked file's thumbnail is embedded once into an [`Index`] -//! on disk; a query is embedded as text, or taken from another image for "find similar", and rows +//! lands near photographs of one. Every linked file's thumbnail is embedded once and stored in the +//! project; a query is embedded as text, or taken from another image for "find similar", and rows //! are ranked by how close their vectors are. //! //! The model runs on the CPU through candle, which is pure Rust. Its weights are not part of the //! install: [`download`] fetches a pinned revision and checks it against the published checksum. -mod index; mod tokenizer; use std::fs; @@ -21,8 +20,6 @@ use candle_transformers::models::clip::{ClipConfig, ClipModel, div_l2_norm}; use image::{DynamicImage, RgbaImage, imageops::FilterType}; use sha2::{Digest as _, Sha256}; -pub use index::{Index, Stamp}; - /// Which weights an index was built with. Vectors from different models are not comparable, so an /// index recorded under another name is discarded rather than searched. pub const MODEL: &str = "openai/clip-vit-base-patch32@b33cedf"; From be56edfcc032551a5732276edc73c4a73e9f5f6d Mon Sep 17 00:00:00 2001 From: devnull03 <56480041+devnull03@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:59:01 -0700 Subject: [PATCH 07/12] feat(workspace): step to the previous or next row from the viewer Up and down buttons, and the arrow keys, open the next row in the view's order that has something to preview, and select it. During a search that is the next result. --- crates/workspace/src/viewer/mod.rs | 95 +++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 1 deletion(-) diff --git a/crates/workspace/src/viewer/mod.rs b/crates/workspace/src/viewer/mod.rs index c4ce9aee..8807ceec 100644 --- a/crates/workspace/src/viewer/mod.rs +++ b/crates/workspace/src/viewer/mod.rs @@ -131,6 +131,65 @@ pub fn open_viewer(path: PathBuf, scope: Scope, window: &mut Window, cx: &mut Ap }); } +/// Open the file of the next row, by `delta`, in the view's order from the one showing, and select +/// that row. During a search the view is its hits, so this steps through the results. Rows with +/// nothing to preview are skipped. +fn step_row(path: &std::path::Path, scope: Scope, delta: isize, window: &mut Window, cx: &mut App) { + let Some(state) = cx + .try_global::() + .and_then(|handle| handle.0.upgrade()) + else { + return; + }; + let target = { + let delegate = state.read(cx).delegate(); + let visible = delegate.visible(); + let file = |view: usize| { + delegate + .row_image(visible[view]) + .filter(|file| preview::can_preview(file)) + }; + let selected = match delegate.selection() { + Some(table::Selection::Cell { row, .. } | table::Selection::Row(row)) => delegate + .view_row(row) + .filter(|&view| file(view) == Some(path)), + _ => None, + }; + selected + .or_else(|| (0..visible.len()).find(|&view| file(view) == Some(path))) + .and_then(|from| next_row(from, delta, visible.len(), |view| file(view).is_some())) + .and_then(|view| Some((view, file(view)?.to_path_buf()))) + }; + let Some((view, file)) = target else { + return; + }; + state.update(cx, |state, cx| { + state.delegate_mut().clear_selection(); + state.set_selected_row(view, cx); + }); + open_viewer(file, scope, window, cx); +} + +/// The nearest view index past `from` in `delta`'s direction that `viewable` accepts, if any. +fn next_row( + from: usize, + delta: isize, + len: usize, + viewable: impl Fn(usize) -> bool, +) -> Option { + let step = delta.signum(); + let mut view = from as isize; + loop { + view += step; + if step == 0 || view < 0 || view >= len as isize { + return None; + } + if viewable(view as usize) { + return Some(view as usize); + } + } +} + pub fn close_viewer(window: &mut Window, cx: &mut App) { // Without this the recording plays on over an empty screen, with nothing left to stop it. preview::playback::stop(cx); @@ -456,6 +515,10 @@ impl Render for Viewer { this.set_zoom(1.0); cx.notify(); } + "up" | "down" if reading => { + let delta = if ev.keystroke.key == "up" { -1 } else { 1 }; + step_row(&this.path, this.scope, delta, window, cx); + } _ => {} } })) @@ -739,6 +802,26 @@ impl Render for Viewer { })), ) }) + .child( + Button::new("previous-row") + .icon(IconName::ChevronUp) + .ghost() + .small() + .tooltip("Previous row (↑)") + .on_click(cx.listener(|this, _, window, cx| { + step_row(&this.path, this.scope, -1, window, cx) + })), + ) + .child( + Button::new("next-row") + .icon(IconName::ChevronDown) + .ghost() + .small() + .tooltip("Next row (↓)") + .on_click(cx.listener(|this, _, window, cx| { + step_row(&this.path, this.scope, 1, window, cx) + })), + ) .child( Button::new("zoom-out") .icon(IconName::Minus) @@ -781,7 +864,17 @@ mod tests { VisualTestContext, Window, div, }; - use crate::viewer::{Scope, close_viewer, open_viewer, viewer_in}; + use crate::viewer::{Scope, close_viewer, next_row, open_viewer, viewer_in}; + + #[test] + fn stepping_rows_skips_what_cannot_be_previewed_and_stops_at_the_ends() { + let viewable = |view: usize| view != 2; + assert_eq!(next_row(1, 1, 5, viewable), Some(3)); + assert_eq!(next_row(3, -1, 5, viewable), Some(1)); + assert_eq!(next_row(4, 1, 5, viewable), None); + assert_eq!(next_row(0, -1, 5, viewable), None); + assert_eq!(next_row(3, 1, 4, viewable), None); + } struct FocusProbe(FocusHandle); From 3d1f74468e548b2c1ffe9ca82240ecb027acaa6e Mon Sep 17 00:00:00 2001 From: devnull03 <56480041+devnull03@users.noreply.github.com> Date: Tue, 15 Sep 2026 14:41:23 -0700 Subject: [PATCH 08/12] fix(workspace): follow the selection in the viewer, with row arrows bottom right The viewer now opens the selected row's file whenever the selection moves, so stepping through search results from the find bar changes the image. The row buttons move from the top toolbar, where they repeated the find bar's, to left and right arrows in the bottom-right corner. --- crates/workspace/src/viewer/mod.rs | 130 ++++++++++++++++++----------- 1 file changed, 83 insertions(+), 47 deletions(-) diff --git a/crates/workspace/src/viewer/mod.rs b/crates/workspace/src/viewer/mod.rs index 8807ceec..1e07349a 100644 --- a/crates/workspace/src/viewer/mod.rs +++ b/crates/workspace/src/viewer/mod.rs @@ -87,7 +87,28 @@ pub fn open_viewer(path: PathBuf, scope: Scope, window: &mut Window, cx: &mut Ap .try_global::() .and_then(|active| active.return_focus.clone()) .or_else(|| window.focused(cx)); + let table = cx + .try_global::() + .and_then(|handle| handle.0.upgrade()); let viewer = cx.new(|cx| Viewer { + _follow: table.map(|table| { + cx.subscribe_in( + &table, + window, + |this: &mut Viewer, table, _: &table::TableChanged, window, cx| { + let delegate = table.read(cx).delegate(); + let file = match delegate.selection() { + Some(table::Selection::Cell { row, .. } | table::Selection::Row(row)) => { + previewable(delegate, row) + } + _ => None, + }; + if let Some(file) = file.filter(|file| *file != this.path) { + open_viewer(file, this.scope, window, cx); + } + }, + ) + }), transport: Transport::new(path.clone(), cx), path, details, @@ -131,10 +152,10 @@ pub fn open_viewer(path: PathBuf, scope: Scope, window: &mut Window, cx: &mut Ap }); } -/// Open the file of the next row, by `delta`, in the view's order from the one showing, and select -/// that row. During a search the view is its hits, so this steps through the results. Rows with -/// nothing to preview are skipped. -fn step_row(path: &std::path::Path, scope: Scope, delta: isize, window: &mut Window, cx: &mut App) { +/// Select the next row, by `delta`, in the view's order that has something to preview. During a +/// search the view is its hits, so this steps through the results. The open viewer follows the +/// selection to that row's file. +fn step_row(delta: isize, cx: &mut App) { let Some(state) = cx .try_global::() .and_then(|handle| handle.0.upgrade()) @@ -144,30 +165,32 @@ fn step_row(path: &std::path::Path, scope: Scope, delta: isize, window: &mut Win let target = { let delegate = state.read(cx).delegate(); let visible = delegate.visible(); - let file = |view: usize| { - delegate - .row_image(visible[view]) - .filter(|file| preview::can_preview(file)) - }; - let selected = match delegate.selection() { - Some(table::Selection::Cell { row, .. } | table::Selection::Row(row)) => delegate - .view_row(row) - .filter(|&view| file(view) == Some(path)), + let from = match delegate.selection() { + Some(table::Selection::Cell { row, .. } | table::Selection::Row(row)) => { + delegate.view_row(row) + } _ => None, }; - selected - .or_else(|| (0..visible.len()).find(|&view| file(view) == Some(path))) - .and_then(|from| next_row(from, delta, visible.len(), |view| file(view).is_some())) - .and_then(|view| Some((view, file(view)?.to_path_buf()))) - }; - let Some((view, file)) = target else { - return; + from.and_then(|from| { + next_row(from, delta, visible.len(), |view| { + previewable(delegate, visible[view]).is_some() + }) + }) }; - state.update(cx, |state, cx| { - state.delegate_mut().clear_selection(); - state.set_selected_row(view, cx); - }); - open_viewer(file, scope, window, cx); + if let Some(view) = target { + state.update(cx, |state, cx| { + state.delegate_mut().clear_selection(); + state.set_selected_row(view, cx); + }); + } +} + +/// The file `row` links to, if the viewer can show it. +fn previewable(delegate: &table::QrateTableDelegate, row: usize) -> Option { + delegate + .row_image(row) + .filter(|file| preview::can_preview(file)) + .map(|file| file.to_path_buf()) } /// The nearest view index past `from` in `delta`'s direction that `viewable` accepts, if any. @@ -242,6 +265,9 @@ pub struct Viewer { /// carries the drag handle, the sizing and the propagation rules, none of which are ours to /// reinvent. split: Entity, + /// Swaps in the selected row's file when the selection moves, from the find bar, the arrows or + /// anywhere else. + _follow: Option, } impl Viewer { @@ -516,8 +542,7 @@ impl Render for Viewer { cx.notify(); } "up" | "down" if reading => { - let delta = if ev.keystroke.key == "up" { -1 } else { 1 }; - step_row(&this.path, this.scope, delta, window, cx); + step_row(if ev.keystroke.key == "up" { -1 } else { 1 }, cx); } _ => {} } @@ -767,6 +792,37 @@ impl Render for Viewer { ) }, ) + // Rows, not pages: the neighbouring files in the view's order, which during a search are + // the neighbouring results. + .child( + div() + .absolute() + .bottom_4() + .right_4() + .flex() + .gap_1() + .p_1() + .rounded(cx.theme().radius) + .bg(cx.theme().background) + .border_1() + .border_color(cx.theme().border) + .shadow_lg() + .occlude() + .child( + Button::new("previous-row") + .icon(IconName::ArrowLeft) + .outline() + .tooltip("Previous row (↑)") + .on_click(|_, _, cx| step_row(-1, cx)), + ) + .child( + Button::new("next-row") + .icon(IconName::ArrowRight) + .outline() + .tooltip("Next row (↓)") + .on_click(|_, _, cx| step_row(1, cx)), + ), + ) .child( div() .absolute() @@ -802,26 +858,6 @@ impl Render for Viewer { })), ) }) - .child( - Button::new("previous-row") - .icon(IconName::ChevronUp) - .ghost() - .small() - .tooltip("Previous row (↑)") - .on_click(cx.listener(|this, _, window, cx| { - step_row(&this.path, this.scope, -1, window, cx) - })), - ) - .child( - Button::new("next-row") - .icon(IconName::ChevronDown) - .ghost() - .small() - .tooltip("Next row (↓)") - .on_click(cx.listener(|this, _, window, cx| { - step_row(&this.path, this.scope, 1, window, cx) - })), - ) .child( Button::new("zoom-out") .icon(IconName::Minus) From 96a7900606a8c129287f14665a838c8ec58023bf Mon Sep 17 00:00:00 2001 From: devnull03 <56480041+devnull03@users.noreply.github.com> Date: Tue, 15 Sep 2026 14:45:14 -0700 Subject: [PATCH 09/12] fix(preview): stop freeing images mid-frame, which crashed fast scrolling The thumbnail memory cap called drop_image from the image loader, which gpui runs while painting. Freeing an image there could release a GPU texture a sprite earlier in the same frame still pointed at, and presenting that frame panicked in the DirectX atlas. The cap was also first-in-first-out, so a card on screen could be the one freed. Images now move to the back of the queue every time they are drawn, and releasing waits for the start of the next frame, then repaints every window so no cached view replays a freed sprite. --- crates/preview/src/lib.rs | 93 +++++++++++++++++++++++++++++---------- 1 file changed, 69 insertions(+), 24 deletions(-) diff --git a/crates/preview/src/lib.rs b/crates/preview/src/lib.rs index 984fb6b8..cb2678e8 100644 --- a/crates/preview/src/lib.rs +++ b/crates/preview/src/lib.rs @@ -54,10 +54,9 @@ const FULL_FALLBACK: u32 = 2048; /// How much decoded image data may stay resident. gpui's asset cache never evicts on its own, so /// without a ceiling a scroll through a large collection retains every thumbnail it passes. /// -/// ponytail: one global budget, FIFO, and a linear scan of the held entries per drawn card. It -/// only has to exceed what is on screen at once — a gallery shows a few dozen cards — so eviction -/// never touches a visible card and true LRU buys nothing. The scan is a few thousand path -/// comparisons a frame at this budget; index the deque by key if the budget is ever raised much. +/// ponytail: one global budget, least recently drawn first, and a linear scan of the held entries +/// per drawn card. The scan is a few thousand path comparisons a frame at this budget; index the +/// deque by key if the budget is ever raised much. const BUDGET: usize = 192 * 1024 * 1024; /// Formats the `image` crate decodes for us directly — gpui's own list minus AVIF, which @@ -513,7 +512,7 @@ fn downscale(image: image::DynamicImage, max_edge: u32) -> image::RgbaImage { .to_rgba8() } -/// Everything currently decoded, oldest first, and what it costs. +/// Everything currently decoded, least recently drawn first, and what it costs. /// /// Holding the `Arc` here is deliberate: releasing an image needs the handle itself, not just its /// cache slot, because the GPU-side texture is freed by `App::drop_image` rather than by the last @@ -522,13 +521,25 @@ fn downscale(image: image::DynamicImage, max_edge: u32) -> image::RgbaImage { struct Live { entries: VecDeque<((PathBuf, u32, usize), Arc)>, bytes: usize, + /// Whether a [`release`] is already queued for the next frame. + releasing: bool, } impl Global for Live {} -/// Account for a freshly-loaded image and release older ones until the total is back under -/// `budget`. Called on every frame a preview is drawn, so re-registering an image already tracked -/// has to be free. +fn cost(image: &RenderImage) -> usize { + // ponytail: frame 0 only, so an animated GIF is undercounted. Costs accuracy on a format the + // budget already tolerates; revisit if animations become common in collections. + image.as_bytes(0).map_or(0, <[u8]>::len) +} + +/// Account for an image being drawn, and queue a [`release`] when the total is over `budget`. +/// Called every time a preview is laid out or painted, so an image already tracked only moves to +/// the back: what is on screen is always the last thing released. +/// +/// Never releases anything itself. This runs inside a frame, and freeing an image there pulls its +/// GPU texture out from under a sprite the same frame already painted, which gpui panics on when +/// it presents the frame. /// /// `budget` is a parameter rather than reading [`BUDGET`] directly so a test can drive eviction /// with a handful of small images instead of allocating its way past the real ceiling. @@ -539,28 +550,50 @@ fn retain( window: &mut Window, cx: &mut App, ) { - // ponytail: frame 0 only, so an animated GIF is undercounted. Costs accuracy on a format the - // budget already tolerates; revisit if animations become common in collections. - let cost = image.as_bytes(0).map_or(0, <[u8]>::len); let live = cx.default_global::(); - if live.entries.iter().any(|(key, _)| key == source) { - return; + match live.entries.iter().position(|(key, _)| key == source) { + Some(at) => { + if let Some(entry) = live.entries.remove(at) { + live.entries.push_back(entry); + } + } + None => { + live.entries.push_back((source.clone(), image.clone())); + live.bytes += cost(image); + } + } + if live.bytes > budget && !live.releasing { + live.releasing = true; + window.on_next_frame(move |window, cx| release(budget, window, cx)); } - live.entries.push_back((source.clone(), image.clone())); - live.bytes += cost; +} - // Never evict the entry just added, or a single oversized image would thrash forever. - while cx.global::().bytes > budget && cx.global::().entries.len() > 1 { - let live = cx.global_mut::(); +/// Free the least recently drawn images until the total is back under `budget`. Runs at the start +/// of a frame, before anything is laid out, so no sprite that frame paints can point at a freed +/// texture. +fn release(budget: usize, window: &mut Window, cx: &mut App) { + let live = cx.default_global::(); + live.releasing = false; + let mut released = Vec::new(); + // Never the most recent entry, or a single oversized image would thrash forever. + while live.bytes > budget && live.entries.len() > 1 { let Some((key, image)) = live.entries.pop_front() else { break; }; - live.bytes = live - .bytes - .saturating_sub(image.as_bytes(0).map_or(0, <[u8]>::len)); + live.bytes = live.bytes.saturating_sub(cost(&image)); + released.push((key, image)); + } + if released.is_empty() { + return; + } + for (key, image) in released { cx.remove_asset::(&key); cx.drop_image(image, Some(window)); } + // A cached view replays the sprites of its last paint without painting again, and those may + // be the images just freed. Repaint everything so no frame presents one. + window.refresh(); + cx.refresh_windows(); } /// The file's contents fit to whatever box the caller gives it, or a type icon when there is @@ -1024,6 +1057,17 @@ mod tests { let key = (PathBuf::from(format!("/f/{n}.png")), crate::CARD, 0); crate::retain(&key, &image(), budget, window, cx); } + assert_eq!( + cx.global::().entries.len(), + 8, + "nothing is freed inside the frame that drew it" + ); + + // Drawn again, so it is no longer the least recently drawn. + let first = cx.global::().entries[0].clone(); + crate::retain(&first.0, &first.1, budget, window, cx); + + crate::release(budget, window, cx); let live = cx.global::(); assert!( live.bytes <= budget, @@ -1031,10 +1075,11 @@ mod tests { live.bytes ); assert_eq!(live.entries.len(), 3, "older images were released"); + let kept: Vec = live.entries.iter().map(|(key, _)| key.0.clone()).collect(); assert_eq!( - live.entries.front().map(|(key, _)| key.0.clone()), - Some(PathBuf::from("/f/5.png")), - "the survivors are the most recent, oldest first" + kept, + ["/f/6.png", "/f/7.png", "/f/0.png"].map(PathBuf::from), + "the survivors are the most recently drawn, in that order" ); // Re-registering something already held must not double-count it: this runs on every From 4042d2d970f8b4558a39fce9e01de8ed86731e99 Mon Sep 17 00:00:00 2001 From: devnull03 <56480041+devnull03@users.noreply.github.com> Date: Tue, 15 Sep 2026 15:14:01 -0700 Subject: [PATCH 10/12] fix: full screen viewer next/prev arrow redesign --- crates/workspace/src/viewer/mod.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/crates/workspace/src/viewer/mod.rs b/crates/workspace/src/viewer/mod.rs index 1e07349a..4732eac5 100644 --- a/crates/workspace/src/viewer/mod.rs +++ b/crates/workspace/src/viewer/mod.rs @@ -803,22 +803,20 @@ impl Render for Viewer { .gap_1() .p_1() .rounded(cx.theme().radius) - .bg(cx.theme().background) - .border_1() - .border_color(cx.theme().border) - .shadow_lg() - .occlude() + .bg(pill) .child( Button::new("previous-row") .icon(IconName::ArrowLeft) - .outline() + .ghost() + .small() .tooltip("Previous row (↑)") .on_click(|_, _, cx| step_row(-1, cx)), ) .child( Button::new("next-row") .icon(IconName::ArrowRight) - .outline() + .ghost() + .small() .tooltip("Next row (↓)") .on_click(|_, _, cx| step_row(1, cx)), ), From a0e59ee656337d9696dff1a17c85fa16bfd802b2 Mon Sep 17 00:00:00 2001 From: devnull03 <56480041+devnull03@users.noreply.github.com> Date: Tue, 15 Sep 2026 15:22:45 -0700 Subject: [PATCH 11/12] fix(settings): read visual vectors with as_chunks for newer clippy Also gives the viewer's row arrows the same ghost buttons as its zoom controls. --- crates/settings/src/project.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/settings/src/project.rs b/crates/settings/src/project.rs index dfa8c1b1..cc8336a2 100644 --- a/crates/settings/src/project.rs +++ b/crates/settings/src/project.rs @@ -711,8 +711,10 @@ pub fn read_visual_index(path: &Path, model: &str) -> Result> { let rows = stmt.query_map([], |r| { let bytes: Vec = r.get(2)?; let vector = bytes - .chunks_exact(4) - .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .as_chunks::<4>() + .0 + .iter() + .map(|b| f32::from_le_bytes(*b)) .collect(); Ok((r.get(0)?, r.get::<_, i64>(1)? as u64, vector)) })?; From 7d542b52a4bcf57ee28b2b670490ac044cdcf56f Mon Sep 17 00:00:00 2001 From: devnull03 <56480041+devnull03@users.noreply.github.com> Date: Sat, 19 Sep 2026 15:22:45 -0700 Subject: [PATCH 12/12] docs: describe visual search --- README.md | 4 ++ docs/files-and-photos.md | 4 ++ docs/grid.md | 9 +++- docs/index.md | 1 + docs/visual-search.md | 111 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 docs/visual-search.md diff --git a/README.md b/README.md index 3a007fec..74c16083 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Collection data rarely lives in just one place. A spreadsheet names an object, a - **Keep ownership of your work.** Each project is a portable `.qrate` SQLite file; linked media remains where you keep it. qrate does not require an account or a hosted service. - **Describe collections with less friction.** Import CSV, Excel, or OpenDocument spreadsheets with their folders. You can also start from a Google Sheet. Edit, search, filter, copy and paste, undo changes, and tailor columns to the collection. - **See the records and the material together.** Link by filename or a pattern, browse a gallery of thumbnails, and preview images, documents, audio, and video alongside each record. +- **Find pictures by what they show.** Optional visual search ranks linked images, PDF pages, and video frames against a plain description, or against another record's picture. The model downloads once and runs on your own machine. - **Catch problems while you work.** Check spelling, date formats, file links, headings, and selected authority sources. Review a proposed whole-cell correction before applying it. - **Take your data where it needs to go.** Export CSV, JSON-LD, CSL-JSON, or a ZIP archive. Google Sheets export and sync are available when you choose to enable them. - **Use AI with a human in control.** An optional local agent can review the open project and stage findings in the Problems panel. It cannot change a cell; you decide what to accept. @@ -59,6 +60,8 @@ PDF preview support is included in release builds. For video previews and some l A `.qrate` file stores the collection grid, settings, notes, and project metadata. Linked media is not copied into it, so you remain in charge of where those files live. +Visual search is off until you download its model, and it runs entirely on your machine: no image and no search text is sent anywhere. The model is stored with qrate's data, and each project keeps its own index inside its `.qrate` file. + qrate works without an account. Google Sheets integration is off until you enable it in **Settings ▸ Google**; sign-in happens on your machine, and qrate only accesses sheets it creates or you select. Plugins run locally—install them only from sources you trust. ## Learn more @@ -66,6 +69,7 @@ qrate works without an account. Google Sheets integration is off until you enabl - [Projects](docs/projects.md) — create, import, and open projects - [The grid](docs/grid.md) — edit, search, filter, and undo - [Files and photos](docs/files-and-photos.md) — link and view collection material +- [Visual search](docs/visual-search.md) — find records by what their pictures show - [Diagnostics](docs/diagnostics.md) — checks, problems, and fixes - [Columns](docs/columns.md) — types, authority lists, and project settings - [Export and Google Sheets](docs/export-and-sync.md) — move data in and out diff --git a/docs/files-and-photos.md b/docs/files-and-photos.md index 890df8d1..539316dc 100644 --- a/docs/files-and-photos.md +++ b/docs/files-and-photos.md @@ -24,6 +24,10 @@ previews images, documents, audio, and video directly. Click the preview to open fullscreen, where you can zoom, pan, page through a multi-page document, and search inside it. +The arrows in the bottom-right corner of the fullscreen view move to the file of the +previous or next row, in the order the view shows them. During a search that means the +previous or next result. + PDF previews need PDFium and video frame previews need ffmpeg. qrate looks for both beside its own executable, then on your system `PATH`. Without them, qrate shows a file-type icon instead of a preview, and the rest of the app works as normal. diff --git a/docs/grid.md b/docs/grid.md index 7c8ed763..da0c71cd 100644 --- a/docs/grid.md +++ b/docs/grid.md @@ -21,7 +21,14 @@ it. A frozen row or column stays in view while you scroll. ## Search and replace Open search with **Ctrl+F**. Search moves through matching cells in the grid. Open replace -with **Ctrl+H** to replace one match or all matches at once. +with **Ctrl+H** to replace one match or all matches at once. The search bar belongs to +whichever view is showing, so a search carries on when you switch between the grid and the +gallery. + +Two toggles in the search bar look beyond the cells, and both narrow the view to the rows +they match. The book searches the text inside linked PDFs. The picture searches what the +linked images show, without using their metadata at all. See +[Visual search](visual-search.md). ## Filtering diff --git a/docs/index.md b/docs/index.md index 7f95bb45..178102a6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,6 +8,7 @@ checks the data against validators, and exports it to the formats other systems - [Projects](projects.md) — create, import, and open a `.qrate` project - [The grid](grid.md) — edit cells, search, filter, and undo - [Files and photos](files-and-photos.md) — link records to files, and view them +- [Visual search](visual-search.md) — find records by what their pictures show - [Diagnostics](diagnostics.md) — the Problems panel, spelling, and fixes - [Columns](columns.md) — column types, authority lists, and per-column settings - [Export and Google Sheets](export-and-sync.md) — CSV, JSON-LD, CSL-JSON, ZIP, and Sheets sync diff --git a/docs/visual-search.md b/docs/visual-search.md new file mode 100644 index 00000000..e9332131 --- /dev/null +++ b/docs/visual-search.md @@ -0,0 +1,111 @@ +# Visual search + +Visual search finds records by what their pictures show, rather than by what their +metadata says. Search for `a wooden bridge` and qrate ranks the collection's images by how +well each one matches that description, even when no cell holds any of those words. + +It also answers the other direction: pick a record and ask for the items that look like +it. + +Everything runs on your own computer. No image and no search text leaves it. + +## What it can search + +Visual search reads the same thumbnails the gallery shows, so it covers every file type +qrate can preview: + +- Photographs and other images. +- PDFs and documents, by their first page. +- Video, by its poster frame. +- Audio, by its cover art, if it has any. + +A row with no linked file cannot be found this way. See +[Files and photos](files-and-photos.md) for how rows link to files. + +## Installing the model + +The first time you turn visual search on, qrate offers a **Download model** button in the +search bar. The download is about 605 MB and happens once per computer, not once per +project. qrate checks the download against a published checksum before installing it. + +The model is stored beside qrate's other data, never inside a project, and it is shared by +every project you open. + +## Searching + +1. Open search with **Ctrl+F**. +2. Turn on the picture toggle in the search bar. +3. Type what you are looking for, in plain words: `a group of people outside a building`. + +While a visual search is running: + +- **The view shows only the results**, best match first. This applies to both the grid and + the gallery, and the search stays as you switch between them. +- **The slider** beside the result count widens or narrows the results. Drag it left for + only the closest matches and right to include weaker ones. Results are scored relative to + the best match, so the slider means the same thing for any search. +- **The arrows** step through the results in order. If a file is open fullscreen, stepping + changes the file it shows. +- **Replace is switched off**, along with match case, whole word and regular expressions. + None of them mean anything for a picture. + +Clear the search box or close the search bar to bring every row back. + +### Writing a good query + +- **Describe the picture, not the record.** `a horse-drawn cart on a dirt road` works; + `accession 1994.12` does not. +- **Plain descriptions beat keywords.** `two people shaking hands` beats `handshake photo + formal`. +- **Expect near misses.** The model ranks every image, so weaker matches follow the good + ones. Tighten the slider rather than reading further down. +- **It reads the picture, not the text in it.** Searching for words written on a document + is a job for the linked-file text search, below. + +## Find similar items + +Right-click a row, or a gallery tile, and choose **Find similar items**. qrate ranks every +other file by how much it looks like that one, and narrows the view to the closest. + +This is how you find the rest of a photo shoot, the other copies of a form, or the pages +that came from the same album. + +## Indexing + +Before a file can be found, qrate reads its thumbnail once and stores what the model made +of it. This is the indexing step, and the search bar shows its progress. + +- **It takes about a tenth of a second per file**, so a few thousand files take a few + minutes the first time. +- **The results are stored in the project file**, so the next time you open the project + there is nothing to redo. +- **A file is read again only if it changes.** Copying a project and its files to another + computer keeps the index. +- **It runs in the background.** You can keep working, though the app may feel slower while + it works through a large collection for the first time. + +If you change the files folder or add new files, only the new files are read. + +## Searching inside linked files + +The book toggle in the search bar, beside the picture one, searches the *text inside* +linked PDFs rather than their appearance. It is off by default. Turn it on and an ordinary +search also matches words in a linked document, narrowing the view to the rows whose files +contain them. + +- The text of each document is read once and kept, so later searches are immediate. +- A hit lands on the cell that names the file. +- Replace does not change text inside a file. It only rewrites cells. +- A scan with no text layer has nothing to search. qrate does not yet read text from + scanned images. + +## Limits + +- **It is a ranking, not a verdict.** The model has no idea what your collection is about, + and it can be confidently wrong. Treat a result as a suggestion to check. +- **It reads one page or frame per file.** Page 40 of a PDF is not searched. +- **It does not read handwriting or printed text.** +- **Terms are English.** The model was trained on English descriptions, so queries in other + languages match poorly. +- **Nothing is written to your data.** Visual search only finds and orders rows. It never + fills a cell in.