diff --git a/CLAUDE.md b/CLAUDE.md index 5cfbba66..ede29088 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -83,6 +83,7 @@ problem — read the `--check` diff and apply it by hand rather than fighting th | `plugin-host` | the Luau runtime that loads and runs plugins | | `plugin-api` | the types a plugin sees — see the three-repo rule below | | `preview` | turns a linked file into pixels — the format ladder and the thumbnail cache. Native decoders (PDF, video, RAW) belong here so they never reach `table` | +| `visual-search` | CLIP on the CPU through candle (pure Rust) and its pinned weights download. No gpui — `table::visual` owns the index job, the search bar, and "Find similar items", and stores the vectors in the project's `.qrate` (`__visual_index`). Weights live in `/models`, never in the install | | `ai` | two halves: `agent.rs` is the *shipped* external-agent contract (see `AGENTS.md`); the rest is traits + Cohere/mock providers for planned AI review/embedding, deliberately unfinished | ## Project Status Tracking diff --git a/Cargo.lock b/Cargo.lock index aeb059e7..2fe53dc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -981,6 +981,64 @@ dependencies = [ "wayland-client", ] +[[package]] +name = "candle-core" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c15b675b80d994b2eadb20a4bbe434eabeb454eac3ee5e2b4cf6f147ee9be091" +dependencies = [ + "byteorder", + "float8", + "gemm", + "half", + "libm", + "memmap2", + "num-traits", + "num_cpus", + "rand 0.9.2", + "rand_distr", + "rayon", + "safetensors", + "thiserror 2.0.18", + "yoke", + "zip 7.2.0", +] + +[[package]] +name = "candle-nn" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045fa9e7aef8567d209a27d56b692f60b96f4d0569f4c3011f8ca6715c65e03" +dependencies = [ + "candle-core", + "half", + "libc", + "num-traits", + "rayon", + "safetensors", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "candle-transformers" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b538ec4aa807c416a2ddd3621044888f188827862e2a6fcacba4738e89795d01" +dependencies = [ + "byteorder", + "candle-core", + "candle-nn", + "fancy-regex 0.17.0", + "num-traits", + "rand 0.9.2", + "rayon", + "serde", + "serde_json", + "serde_plain", + "tracing", +] + [[package]] name = "caseless" version = "0.2.2" @@ -1942,6 +2000,22 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "dyn-stack" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c4713e43e2886ba72b8271aa66c93d722116acf7a75555cce11dcde84388fe8" +dependencies = [ + "bytemuck", + "dyn-stack-macros", +] + +[[package]] +name = "dyn-stack-macros" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d926b4d407d372f141f93bb444696142c29d32962ccbd3531117cf3aa0bfa9" + [[package]] name = "ed25519" version = "2.2.3" @@ -2011,6 +2085,18 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099" +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "enum-iterator" version = "2.3.0" @@ -2194,6 +2280,17 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "fancy-regex" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" +dependencies = [ + "bit-set 0.8.0", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fast-float2" version = "0.2.4" @@ -2301,6 +2398,18 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" +[[package]] +name = "float8" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719a903cc23e4a89e87962c2a80fdb45cdaad0983a89bd150bb57b4c8571a7d5" +dependencies = [ + "half", + "num-traits", + "rand 0.9.2", + "rand_distr", +] + [[package]] name = "float_next_after" version = "1.0.0" @@ -2582,6 +2691,125 @@ dependencies = [ "slab", ] +[[package]] +name = "gemm" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa0673db364b12263d103b68337a68fbecc541d6f6b61ba72fe438654709eacb" +dependencies = [ + "dyn-stack", + "gemm-c32", + "gemm-c64", + "gemm-common", + "gemm-f16", + "gemm-f32", + "gemm-f64", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-c32" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "086936dbdcb99e37aad81d320f98f670e53c1e55a98bee70573e83f95beb128c" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-c64" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20c8aeeeec425959bda4d9827664029ba1501a90a0d1e6228e48bef741db3a3f" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-common" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88027625910cc9b1085aaaa1c4bc46bb3a36aad323452b33c25b5e4e7c8e2a3e" +dependencies = [ + "bytemuck", + "dyn-stack", + "half", + "libm", + "num-complex", + "num-traits", + "once_cell", + "paste", + "pulp", + "raw-cpuid", + "rayon", + "seq-macro", + "sysctl", +] + +[[package]] +name = "gemm-f16" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3df7a55202e6cd6739d82ae3399c8e0c7e1402859b30e4cb780e61525d9486e" +dependencies = [ + "dyn-stack", + "gemm-common", + "gemm-f32", + "half", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "rayon", + "seq-macro", +] + +[[package]] +name = "gemm-f32" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e0b8c9da1fbec6e3e3ab2ce6bc259ef18eb5f6f0d3e4edf54b75f9fd41a81c" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-f64" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "056131e8f2a521bfab322f804ccd652520c79700d81209e9d9275bbdecaadc6a" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -3415,9 +3643,12 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ + "bytemuck", "cfg-if", "crunchy", "num-traits", + "rand 0.9.2", + "rand_distr", "zerocopy", ] @@ -3469,6 +3700,8 @@ dependencies = [ "allocator-api2", "equivalent", "foldhash 0.2.0", + "serde", + "serde_core", ] [[package]] @@ -4576,6 +4809,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" dependencies = [ "libc", + "stable_deref_trait", ] [[package]] @@ -4965,6 +5199,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ + "bytemuck", "num-traits", ] @@ -6194,6 +6429,29 @@ dependencies = [ "syn", ] +[[package]] +name = "pulp" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046aa45b989642ec2e4717c8e72d677b13edd831a4d3b6cf37d9a3e54912496a" +dependencies = [ + "bytemuck", + "cfg-if", + "libm", + "num-complex", + "paste", + "pulp-wasm-simd-flag", + "raw-cpuid", + "reborrow", + "version_check", +] + +[[package]] +name = "pulp-wasm-simd-flag" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d8f70e07b9c3962945a74e59ca1c511bba65b6419468acc217c457d93f3c740" + [[package]] name = "pxfm" version = "0.1.28" @@ -6404,6 +6662,16 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.2", +] + [[package]] name = "rand_xorshift" version = "0.4.0" @@ -6475,6 +6743,15 @@ dependencies = [ "rgb", ] +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.11.0", +] + [[package]] name = "raw-window-handle" version = "0.6.2" @@ -6534,6 +6811,12 @@ dependencies = [ "font-types 0.11.3", ] +[[package]] +name = "reborrow" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" + [[package]] name = "redox_syscall" version = "0.5.18" @@ -7012,6 +7295,17 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safetensors" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "675656c1eabb620b921efea4f9199f97fc86e36dd6ffd1fbbe48d0f59a4987f5" +dependencies = [ + "hashbrown 0.16.1", + "serde", + "serde_json", +] + [[package]] name = "same-file" version = "1.0.6" @@ -7168,6 +7462,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + [[package]] name = "serde" version = "1.0.228" @@ -7252,6 +7552,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_plain" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50" +dependencies = [ + "serde", +] + [[package]] name = "serde_repr" version = "0.1.20" @@ -7989,7 +8298,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "656b45c05d95a5704399aeef6bd0ddec7b2b3531b7c9e900abbf7c4d2190c925" dependencies = [ "bincode", - "fancy-regex", + "fancy-regex 0.16.2", "flate2", "fnv", "once_cell", @@ -8009,6 +8318,20 @@ dependencies = [ "libc", ] +[[package]] +name = "sysctl" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" +dependencies = [ + "bitflags 2.11.0", + "byteorder", + "enum-as-inner", + "libc", + "thiserror 1.0.69", + "walkdir", +] + [[package]] name = "sysinfo" version = "0.31.4" @@ -8089,6 +8412,7 @@ dependencies = [ "serde_json", "settings", "sha2", + "visual-search", ] [[package]] @@ -8643,6 +8967,12 @@ dependencies = [ "core_maths", ] +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typeid" version = "1.0.3" @@ -8965,6 +9295,23 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "visual-search" +version = "0.4.0-beta.2" +dependencies = [ + "anyhow", + "candle-core", + "candle-nn", + "candle-transformers", + "dirs 6.0.0", + "image", + "log", + "regex", + "reqwest", + "serde_json", + "sha2", +] + [[package]] name = "vswhom" version = "0.1.0" @@ -10911,6 +11258,18 @@ dependencies = [ "zopfli", ] +[[package]] +name = "zip" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0" +dependencies = [ + "crc32fast", + "indexmap", + "memchr", + "typed-path", +] + [[package]] name = "zlib-rs" version = "0.6.7" diff --git a/Cargo.toml b/Cargo.toml index b3f3016e..4220c536 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ members = [ "crates/spellcheck", "crates/table", "crates/updater", + "crates/visual-search", "crates/window-wrapper", "crates/workspace", ] diff --git a/README.md b/README.md index 3a007fec..74c16083 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Collection data rarely lives in just one place. A spreadsheet names an object, a - **Keep ownership of your work.** Each project is a portable `.qrate` SQLite file; linked media remains where you keep it. qrate does not require an account or a hosted service. - **Describe collections with less friction.** Import CSV, Excel, or OpenDocument spreadsheets with their folders. You can also start from a Google Sheet. Edit, search, filter, copy and paste, undo changes, and tailor columns to the collection. - **See the records and the material together.** Link by filename or a pattern, browse a gallery of thumbnails, and preview images, documents, audio, and video alongside each record. +- **Find pictures by what they show.** Optional visual search ranks linked images, PDF pages, and video frames against a plain description, or against another record's picture. The model downloads once and runs on your own machine. - **Catch problems while you work.** Check spelling, date formats, file links, headings, and selected authority sources. Review a proposed whole-cell correction before applying it. - **Take your data where it needs to go.** Export CSV, JSON-LD, CSL-JSON, or a ZIP archive. Google Sheets export and sync are available when you choose to enable them. - **Use AI with a human in control.** An optional local agent can review the open project and stage findings in the Problems panel. It cannot change a cell; you decide what to accept. @@ -59,6 +60,8 @@ PDF preview support is included in release builds. For video previews and some l A `.qrate` file stores the collection grid, settings, notes, and project metadata. Linked media is not copied into it, so you remain in charge of where those files live. +Visual search is off until you download its model, and it runs entirely on your machine: no image and no search text is sent anywhere. The model is stored with qrate's data, and each project keeps its own index inside its `.qrate` file. + qrate works without an account. Google Sheets integration is off until you enable it in **Settings ▸ Google**; sign-in happens on your machine, and qrate only accesses sheets it creates or you select. Plugins run locally—install them only from sources you trust. ## Learn more @@ -66,6 +69,7 @@ qrate works without an account. Google Sheets integration is off until you enabl - [Projects](docs/projects.md) — create, import, and open projects - [The grid](docs/grid.md) — edit, search, filter, and undo - [Files and photos](docs/files-and-photos.md) — link and view collection material +- [Visual search](docs/visual-search.md) — find records by what their pictures show - [Diagnostics](docs/diagnostics.md) — checks, problems, and fixes - [Columns](docs/columns.md) — types, authority lists, and project settings - [Export and Google Sheets](docs/export-and-sync.md) — move data in and out diff --git a/assets/icons/image.svg b/assets/icons/image.svg new file mode 100644 index 00000000..135ea222 --- /dev/null +++ b/assets/icons/image.svg @@ -0,0 +1 @@ + diff --git a/crates/app/src/actions.rs b/crates/app/src/actions.rs index 29e9dadc..aa5d4764 100644 --- a/crates/app/src/actions.rs +++ b/crates/app/src/actions.rs @@ -50,11 +50,15 @@ pub fn key_bindings() -> Vec { // registered globally in `main.rs`, so this only adds the key. KeyBinding::new("secondary-,", OpenSettings, None), // The grid's own actions are declared in `crate::table` (app→table is one-way). Ctrl+F is - // scoped to the panel so it works wherever focus sits inside it; everything below acts on a + // scoped to the centre views so it works in Table and Gallery alike; everything below acts on a // grid selection and is scoped to the grid, which keeps the cell editor's own Ctrl+Z/Ctrl+C // and its Enter-to-commit intact while it has focus. - KeyBinding::new("secondary-f", table::Search, Some("TablePanel")), - KeyBinding::new("secondary-h", table::Replace, Some("TablePanel")), + KeyBinding::new("secondary-f", table::Search, Some(workspace::VIEWS_CONTEXT)), + KeyBinding::new( + "secondary-h", + table::Replace, + Some(workspace::VIEWS_CONTEXT), + ), KeyBinding::new("enter", table::EditCell, Some(table::GRID_CONTEXT)), KeyBinding::new("secondary-z", table::Undo, Some(table::GRID_CONTEXT)), KeyBinding::new("secondary-y", table::Redo, Some(table::GRID_CONTEXT)), diff --git a/crates/app/src/assets.rs b/crates/app/src/assets.rs index a626bc69..4b0b17fc 100644 --- a/crates/app/src/assets.rs +++ b/crates/app/src/assets.rs @@ -1,6 +1,6 @@ //! Our own icons in front of `gpui_component_assets`, which is otherwise the only asset source. -//! Lucide ships no filled panel glyph, so the title bar's "this dock is open" state needs three -//! SVGs of our own. +//! The bundled icon set has no filled panel glyphs (the title bar's "this dock is open" state) and +//! no picture glyph (visual search), so those SVGs are ours, copied from Lucide. use std::borrow::Cow; @@ -8,7 +8,7 @@ use gpui::{AssetSource, Result, SharedString}; pub struct Assets; -const OWN: [(&str, &str); 3] = [ +const OWN: [(&str, &str); 4] = [ ( "icons/panel-left-filled.svg", include_str!("../../../assets/icons/panel-left-filled.svg"), @@ -21,6 +21,10 @@ const OWN: [(&str, &str); 3] = [ "icons/panel-bottom-filled.svg", include_str!("../../../assets/icons/panel-bottom-filled.svg"), ), + ( + "icons/image.svg", + include_str!("../../../assets/icons/image.svg"), + ), ]; impl AssetSource for Assets { diff --git a/crates/preview/src/lib.rs b/crates/preview/src/lib.rs index bd31a22b..cb2678e8 100644 --- a/crates/preview/src/lib.rs +++ b/crates/preview/src/lib.rs @@ -54,10 +54,9 @@ const FULL_FALLBACK: u32 = 2048; /// How much decoded image data may stay resident. gpui's asset cache never evicts on its own, so /// without a ceiling a scroll through a large collection retains every thumbnail it passes. /// -/// ponytail: one global budget, FIFO, and a linear scan of the held entries per drawn card. It -/// only has to exceed what is on screen at once — a gallery shows a few dozen cards — so eviction -/// never touches a visible card and true LRU buys nothing. The scan is a few thousand path -/// comparisons a frame at this budget; index the deque by key if the budget is ever raised much. +/// ponytail: one global budget, least recently drawn first, and a linear scan of the held entries +/// per drawn card. The scan is a few thousand path comparisons a frame at this budget; index the +/// deque by key if the budget is ever raised much. const BUDGET: usize = 192 * 1024 * 1024; /// Formats the `image` crate decodes for us directly — gpui's own list minus AVIF, which @@ -346,6 +345,32 @@ pub fn has_text_layer(path: &Path) -> Option { pdf::has_text_layer(path) } +/// The whole text of a document, for searching across a collection rather than inside one file. +/// Cached on disk beside the thumbnails, so a project's documents are read once, not every launch. +/// `None` for anything that is not a document, and where PDFium is not installed. +pub fn document_text(path: &Path) -> Option { + if !has_text(path) { + return None; + } + let entry = cache::key(path, FULL, 0).and_then(|key| Some(cache::dir()?.join(key + ".txt"))); + if let Some(text) = entry + .as_ref() + .and_then(|entry| std::fs::read_to_string(entry).ok()) + { + return Some(text); + } + let text = pdf::text(path)?; + if let Some(entry) = entry + && let Err(err) = std::fs::write(&entry, &text) + { + log::warn!( + "could not cache the text of {}, it will be read again next launch: {err}", + path.display() + ); + } + Some(text) +} + /// Every hit for `needle` in this file, in document order. Empty for a blank query, for anything /// that is not a document, and where PDFium is not installed. pub fn search(path: &Path, needle: &str) -> Vec { @@ -379,7 +404,9 @@ fn render(path: &Path, max_edge: u32, page: usize) -> Option> { Some(Arc::new(RenderImage::new([Frame::new(bgra)]))) } -fn thumbnail_pixels(path: &Path, max_edge: u32, page: usize) -> Option { +/// `path` decoded down the ladder and shrunk to fit `max_edge`, from the disk cache when it has been +/// drawn before. What a card shows, as pixels rather than something only gpui can draw. +pub fn thumbnail_pixels(path: &Path, max_edge: u32, page: usize) -> Option { // A full-size rendering is not cached: it would be a second copy of the original file on disk, // and it is wanted once, while someone is looking at it. let key = (max_edge != FULL) @@ -485,7 +512,7 @@ fn downscale(image: image::DynamicImage, max_edge: u32) -> image::RgbaImage { .to_rgba8() } -/// Everything currently decoded, oldest first, and what it costs. +/// Everything currently decoded, least recently drawn first, and what it costs. /// /// Holding the `Arc` here is deliberate: releasing an image needs the handle itself, not just its /// cache slot, because the GPU-side texture is freed by `App::drop_image` rather than by the last @@ -494,13 +521,25 @@ fn downscale(image: image::DynamicImage, max_edge: u32) -> image::RgbaImage { struct Live { entries: VecDeque<((PathBuf, u32, usize), Arc)>, bytes: usize, + /// Whether a [`release`] is already queued for the next frame. + releasing: bool, } impl Global for Live {} -/// Account for a freshly-loaded image and release older ones until the total is back under -/// `budget`. Called on every frame a preview is drawn, so re-registering an image already tracked -/// has to be free. +fn cost(image: &RenderImage) -> usize { + // ponytail: frame 0 only, so an animated GIF is undercounted. Costs accuracy on a format the + // budget already tolerates; revisit if animations become common in collections. + image.as_bytes(0).map_or(0, <[u8]>::len) +} + +/// Account for an image being drawn, and queue a [`release`] when the total is over `budget`. +/// Called every time a preview is laid out or painted, so an image already tracked only moves to +/// the back: what is on screen is always the last thing released. +/// +/// Never releases anything itself. This runs inside a frame, and freeing an image there pulls its +/// GPU texture out from under a sprite the same frame already painted, which gpui panics on when +/// it presents the frame. /// /// `budget` is a parameter rather than reading [`BUDGET`] directly so a test can drive eviction /// with a handful of small images instead of allocating its way past the real ceiling. @@ -511,28 +550,50 @@ fn retain( window: &mut Window, cx: &mut App, ) { - // ponytail: frame 0 only, so an animated GIF is undercounted. Costs accuracy on a format the - // budget already tolerates; revisit if animations become common in collections. - let cost = image.as_bytes(0).map_or(0, <[u8]>::len); let live = cx.default_global::(); - if live.entries.iter().any(|(key, _)| key == source) { - return; + match live.entries.iter().position(|(key, _)| key == source) { + Some(at) => { + if let Some(entry) = live.entries.remove(at) { + live.entries.push_back(entry); + } + } + None => { + live.entries.push_back((source.clone(), image.clone())); + live.bytes += cost(image); + } } - live.entries.push_back((source.clone(), image.clone())); - live.bytes += cost; + if live.bytes > budget && !live.releasing { + live.releasing = true; + window.on_next_frame(move |window, cx| release(budget, window, cx)); + } +} - // Never evict the entry just added, or a single oversized image would thrash forever. - while cx.global::().bytes > budget && cx.global::().entries.len() > 1 { - let live = cx.global_mut::(); +/// Free the least recently drawn images until the total is back under `budget`. Runs at the start +/// of a frame, before anything is laid out, so no sprite that frame paints can point at a freed +/// texture. +fn release(budget: usize, window: &mut Window, cx: &mut App) { + let live = cx.default_global::(); + live.releasing = false; + let mut released = Vec::new(); + // Never the most recent entry, or a single oversized image would thrash forever. + while live.bytes > budget && live.entries.len() > 1 { let Some((key, image)) = live.entries.pop_front() else { break; }; - live.bytes = live - .bytes - .saturating_sub(image.as_bytes(0).map_or(0, <[u8]>::len)); + live.bytes = live.bytes.saturating_sub(cost(&image)); + released.push((key, image)); + } + if released.is_empty() { + return; + } + for (key, image) in released { cx.remove_asset::(&key); cx.drop_image(image, Some(window)); } + // A cached view replays the sprites of its last paint without painting again, and those may + // be the images just freed. Repaint everything so no frame presents one. + window.refresh(); + cx.refresh_windows(); } /// The file's contents fit to whatever box the caller gives it, or a type icon when there is @@ -996,6 +1057,17 @@ mod tests { let key = (PathBuf::from(format!("/f/{n}.png")), crate::CARD, 0); crate::retain(&key, &image(), budget, window, cx); } + assert_eq!( + cx.global::().entries.len(), + 8, + "nothing is freed inside the frame that drew it" + ); + + // Drawn again, so it is no longer the least recently drawn. + let first = cx.global::().entries[0].clone(); + crate::retain(&first.0, &first.1, budget, window, cx); + + crate::release(budget, window, cx); let live = cx.global::(); assert!( live.bytes <= budget, @@ -1003,10 +1075,11 @@ mod tests { live.bytes ); assert_eq!(live.entries.len(), 3, "older images were released"); + let kept: Vec = live.entries.iter().map(|(key, _)| key.0.clone()).collect(); assert_eq!( - live.entries.front().map(|(key, _)| key.0.clone()), - Some(PathBuf::from("/f/5.png")), - "the survivors are the most recent, oldest first" + kept, + ["/f/6.png", "/f/7.png", "/f/0.png"].map(PathBuf::from), + "the survivors are the most recently drawn, in that order" ); // Re-registering something already held must not double-count it: this runs on every diff --git a/crates/preview/src/pdf.rs b/crates/preview/src/pdf.rs index 59e7a234..354c2415 100644 --- a/crates/preview/src/pdf.rs +++ b/crates/preview/src/pdf.rs @@ -94,6 +94,23 @@ pub fn page_count(path: &Path) -> Option { usize::try_from(document.pages().len()).ok() } +/// The text layer of every page, one page per line break. Empty for a scan that was never OCR'd. +/// +/// ponytail: holds the PDFium lock for the whole document, so a 500-page PDF stalls gallery +/// thumbnails until it is read. Per-page locking if that shows up in a real collection. +pub fn text(path: &Path) -> Option { + let pdfium = locked()?; + let document = pdfium.load_pdf_from_file(path, None).ok()?; + Some( + document + .pages() + .iter() + .filter_map(|page| page.text().ok().map(|text| text.all())) + .collect::>() + .join("\n"), + ) +} + /// Whether the document carries a text layer at all. /// /// A scan that was never OCR'd has none, and "no matches" is a misleading answer to every search @@ -404,6 +421,28 @@ trailer<>"; let _ = std::fs::remove_file(&typed); } + /// Collection search reads every page, and reads it again from the cache on the next launch. + #[test] + fn document_text_covers_every_page_and_survives_the_cache() { + let path = document_with_text("whole"); + let Some(text) = crate::document_text(&path) else { + eprintln!("skipping the document text check: PDFium is not installed"); + return; + }; + assert!(text.contains("Aderman family") && text.contains("Kitsilano beach")); + assert_eq!( + crate::document_text(&path), + Some(text), + "the cached copy reads back" + ); + assert_eq!( + crate::document_text(std::path::Path::new("photo.jpg")), + None + ); + + let _ = std::fs::remove_file(&path); + } + /// Paging has to actually select a different page, not just re-render the first one. The two /// pages are deliberately opposite orientations, which is observable in the result without /// needing any page content to draw. diff --git a/crates/settings/src/project.rs b/crates/settings/src/project.rs index abbbd1af..cc8336a2 100644 --- a/crates/settings/src/project.rs +++ b/crates/settings/src/project.rs @@ -671,6 +671,96 @@ pub fn write_notes(path: &Path, source: &str, notes: &[StoredNote]) -> Result<() Ok(()) } +/// Created on first write, like `__notes`, so opening a project stays read-only. Keyed by the path +/// as the caller stores it (relative to the files folder where it can be). +const VISUAL_INDEX_DDL: &str = r#" + CREATE TABLE IF NOT EXISTS __visual_index ( + path TEXT PRIMARY KEY, + len INTEGER NOT NULL, + vector BLOB NOT NULL + ); +"#; + +/// `__settings` key naming the model that built `__visual_index`. Vectors from different models +/// cannot be compared, so a mismatch reads as an empty index and the next write clears it. +const VISUAL_MODEL_KEY: &str = "visual_model"; + +/// One image's vector: the stored path, the file's length when embedded, and the vector. +pub type VisualEntry = (String, u64, Vec); + +/// Every vector `model` stored in this project. Empty when there is no index yet, or another model +/// built it. +pub fn read_visual_index(path: &Path, model: &str) -> Result> { + let conn = open_ro(path)?; + let exists: i64 = conn.query_row( + "SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name = '__visual_index'", + [], + |r| r.get(0), + )?; + let built_by: Option = conn + .query_row( + "SELECT value FROM __settings WHERE key = ?1", + params![VISUAL_MODEL_KEY], + |r| r.get(0), + ) + .optional()?; + if exists == 0 || built_by.as_deref() != Some(model) { + return Ok(Vec::new()); + } + let mut stmt = conn.prepare("SELECT path, len, vector FROM __visual_index")?; + let rows = stmt.query_map([], |r| { + let bytes: Vec = r.get(2)?; + let vector = bytes + .as_chunks::<4>() + .0 + .iter() + .map(|b| f32::from_le_bytes(*b)) + .collect(); + Ok((r.get(0)?, r.get::<_, i64>(1)? as u64, vector)) + })?; + rows.collect::>() + .context("Read visual index") +} + +/// Store `entries`, replacing any with the same path, in one transaction. Clears what another model +/// stored first. +pub fn write_visual_index(path: &Path, model: &str, entries: &[VisualEntry]) -> Result<()> { + let mut conn = open_rw(path)?; + let tx = conn.transaction()?; + tx.execute_batch(VISUAL_INDEX_DDL) + .context("Create __visual_index")?; + let built_by: Option = tx + .query_row( + "SELECT value FROM __settings WHERE key = ?1", + params![VISUAL_MODEL_KEY], + |r| r.get(0), + ) + .optional()?; + if built_by.as_deref() != Some(model) { + tx.execute("DELETE FROM __visual_index", []) + .context("Clear another model's vectors")?; + tx.execute( + "INSERT INTO __settings(key, value) VALUES (?1, ?2) + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + params![VISUAL_MODEL_KEY, model], + ) + .context("Record visual model")?; + } + { + let mut stmt = tx.prepare( + "INSERT INTO __visual_index(path, len, vector) VALUES (?1, ?2, ?3) + ON CONFLICT(path) DO UPDATE SET len = excluded.len, vector = excluded.vector", + )?; + for (file, len, vector) in entries { + let bytes: Vec = vector.iter().flat_map(|v| v.to_le_bytes()).collect(); + stmt.execute(params![file, *len as i64, bytes]) + .context("Insert vector")?; + } + } + tx.commit().context("Commit visual index")?; + Ok(()) +} + /// Debounced background writer for `__settings` values. The dock-layout and /// window-bounds observers fire on every drag event; latest value per /// (file, key) wins, and one thread serves all project files — no lifecycle @@ -927,6 +1017,26 @@ mod tests { .unwrap(); } + #[test] + fn visual_index_round_trips_and_forgets_another_models_vectors() { + let path = tempfile("visual.qrate"); + blank_project(&path); + assert!(read_visual_index(&path, "clip-a").unwrap().is_empty()); + + write_visual_index(&path, "clip-a", &[("a.jpg".into(), 6, vec![0.6, 0.8])]).unwrap(); + write_visual_index(&path, "clip-a", &[("a.jpg".into(), 7, vec![1.0, 0.0])]).unwrap(); + assert_eq!( + read_visual_index(&path, "clip-a").unwrap(), + vec![("a.jpg".to_string(), 7, vec![1.0, 0.0])], + "a re-embedded file replaces its row" + ); + assert!(read_visual_index(&path, "clip-b").unwrap().is_empty()); + + write_visual_index(&path, "clip-b", &[("b.jpg".into(), 1, vec![0.0])]).unwrap(); + assert_eq!(read_visual_index(&path, "clip-b").unwrap().len(), 1); + assert!(read_visual_index(&path, "clip-a").unwrap().is_empty()); + } + /// A project written before notes carried a date and an author must still open, still read its /// notes, and gain the columns on the next save — losing a catalogue's marginalia to a schema /// change is the one outcome here that cannot be undone. diff --git a/crates/table/Cargo.toml b/crates/table/Cargo.toml index 5896d37e..8360e352 100644 --- a/crates/table/Cargo.toml +++ b/crates/table/Cargo.toml @@ -17,6 +17,7 @@ plugin-api = { path = "../plugin-api" } settings = { path = "../settings" } plugin-host = { path = "../plugin-host" } preview = { path = "../preview" } +visual-search = { path = "../visual-search" } base64 = "0.22" sha2 = "0.10" diff --git a/crates/table/src/delegate.rs b/crates/table/src/delegate.rs index 6e7efc4d..583d82da 100644 --- a/crates/table/src/delegate.rs +++ b/crates/table/src/delegate.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeSet, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::ops::RangeInclusive; use std::path::{Path, PathBuf}; @@ -73,9 +73,17 @@ pub struct QrateTableDelegate { pub(crate) note_editor: Entity, /// Each row's resolved image path, parallel to `rows`. `None` until `TablePanel` resolves it. image_paths: Vec>, + /// The text inside linked documents, by path. Filled by `TablePanel` the first time a search + /// includes linked files; a file with no text layer holds an empty string so it is not re-read. + document_text: HashMap, /// View→source row mapping: `visible_rows[view] == source`. The library only ever sees this /// narrowed set, so filtering composes with the virtualized render for free. visible_rows: Vec, + /// The rows the column filters let through, before a search narrows them. What searches scan, + /// so typing more can widen the hits again. + filtered_rows: Vec, + /// A visual or linked-file search's hits as source rows, in the order the view shows them. + search_rows: Option>, /// Per-data-column set of *excluded* cell values, parallel to `columns`. Empty = no filter. filters: Vec>, /// Whether each column offers a filter dropdown at all, parallel to `columns`. Off for every @@ -112,7 +120,10 @@ impl QrateTableDelegate { note_edit: None, note_editor, image_paths: Vec::new(), + document_text: HashMap::new(), visible_rows: Vec::new(), + filtered_rows: Vec::new(), + search_rows: None, filters: Vec::new(), filters_enabled: Vec::new(), subdelimiter: SharedString::default(), @@ -197,7 +208,17 @@ impl QrateTableDelegate { } fn recompute_visible(&mut self) { - self.visible_rows = compute_visible_rows(&self.rows, &self.filters, &self.subdelimiter); + self.filtered_rows = compute_visible_rows(&self.rows, &self.filters, &self.subdelimiter); + self.visible_rows = match &self.search_rows { + Some(hits) => { + let kept: HashSet = self.filtered_rows.iter().copied().collect(); + hits.iter() + .copied() + .filter(|row| kept.contains(row)) + .collect() + } + None => self.filtered_rows.clone(), + }; // `range` is in view coordinates, which this just redefined. self.range = None; } @@ -321,13 +342,112 @@ impl QrateTableDelegate { /// Cells in the *visible* set matching `needle` under `opts`, as `(view_row, data_col)` in view /// order — ready for `set_selected_cell`/`scroll_to_row` after the pinned `+1` on the column. pub(crate) fn search_matches(&self, needle: &str, opts: SearchOpts) -> Vec<(usize, usize)> { - find_matches( + let mut hits = find_matches( &self.rows, &self.visible_rows, self.columns.len(), needle, opts, - ) + ); + if opts.files { + hits.extend(find_file_matches( + &self.rows, + &self.visible_rows, + &self.image_paths, + &self.document_text, + needle, + opts, + )); + hits.sort_unstable(); + hits.dedup(); + } + hits + } + + /// Every filtered row with a linked file that `score` can rate, as `(score, source_row, + /// data_col)` best first. Ignores any narrowing a previous search applied. + pub(crate) fn ranked_rows( + &self, + score: impl Fn(&Path) -> Option, + ) -> Vec<(f32, usize, usize)> { + let mut scored: Vec<(f32, usize, usize)> = self + .filtered_rows + .iter() + .filter_map(|&source| { + let path = self.image_paths.get(source)?.as_ref()?; + let col = file_column(self.rows.get(source)?, path); + Some((score(path)?, source, col)) + }) + .collect(); + scored.sort_by(|a, b| b.0.total_cmp(&a.0)); + scored + } + + /// The filtered rows with a hit for `needle`, in cells or linked files, as source rows in order. + /// Ignores any narrowing a previous search applied. + pub(crate) fn hit_rows(&self, needle: &str, opts: SearchOpts) -> Vec { + let mut hits = find_matches( + &self.rows, + &self.filtered_rows, + self.columns.len(), + needle, + opts, + ); + if opts.files { + hits.extend(find_file_matches( + &self.rows, + &self.filtered_rows, + &self.image_paths, + &self.document_text, + needle, + opts, + )); + } + let mut rows: Vec = hits + .into_iter() + .map(|(at, _)| self.filtered_rows[at]) + .collect(); + rows.sort_unstable(); + rows.dedup(); + rows + } + + /// Show only `rows`, in that order, until a later call passes `None`. Column filters still apply. + /// Returns whether the view changed. + pub(crate) fn set_search_rows(&mut self, rows: Option>) -> bool { + if self.search_rows == rows { + return false; + } + self.search_rows = rows; + self.recompute_visible(); + self.editing = EditState::Idle; + true + } + + /// Every distinct file the rows link to. + pub(crate) fn linked_files(&self) -> Vec { + let mut files: Vec = self.image_paths.iter().flatten().cloned().collect(); + files.sort_unstable(); + files.dedup(); + files + } + + /// Linked documents whose text has not been read yet, each once. + pub(crate) fn unread_documents(&self) -> Vec { + let mut unread: Vec = self + .image_paths + .iter() + .flatten() + .filter(|path| preview::has_text(path) && !self.document_text.contains_key(*path)) + .cloned() + .collect(); + unread.sort_unstable(); + unread.dedup(); + unread + } + + pub(crate) fn add_document_text(&mut self, texts: Vec<(PathBuf, String)>) { + self.document_text.extend(texts); } /// The cell writes replacing `needle` with `replacement` produces, in *source* coordinates and @@ -388,7 +508,9 @@ impl QrateTableDelegate { self.history = History::default(); self.filters = vec![HashSet::new(); self.columns.len()]; self.filters_enabled = vec![false; self.columns.len()]; - self.visible_rows = (0..self.rows.len()).collect(); + self.search_rows = None; + self.filtered_rows = (0..self.rows.len()).collect(); + self.visible_rows = self.filtered_rows.clone(); // Stale — indexes into the old row set; `TablePanel` re-resolves right after. self.image_paths = vec![None; self.rows.len()]; self.values_generation += 1; @@ -688,6 +810,8 @@ impl QrateTableDelegate { /// What every row insert or delete invalidates. The view is rebuilt rather than patched: a /// filtered view's indices all move, and `recompute_visible` already drops the range for that. fn rows_changed(&mut self) { + // Hits are source rows, which an insert or delete just shifted. + self.search_rows = None; self.recompute_visible(); self.editing = EditState::Idle; self.values_generation += 1; @@ -1070,6 +1194,10 @@ pub(crate) struct SearchOpts { pub case: bool, pub word: bool, pub regex: bool, + /// Also match the text inside each row's linked document. + pub files: bool, + /// Rank rows by how well their linked file matches a description instead of matching text. + pub visual: bool, } /// Compile the query into a matcher honoring the toggles. `None` means "match nothing": a blank @@ -1121,6 +1249,48 @@ fn find_matches( hits } +/// Rows whose linked document contains `needle`, addressed at the cell that names the file so +/// stepping to the hit lands where the archivist would look for it. +fn find_file_matches( + rows: &[Vec], + visible: &[usize], + image_paths: &[Option], + document_text: &HashMap, + needle: &str, + opts: SearchOpts, +) -> Vec<(usize, usize)> { + let Some(re) = compile_search(needle, opts) else { + return Vec::new(); + }; + visible + .iter() + .enumerate() + .filter_map(|(view, &source)| { + let path = image_paths.get(source)?.as_ref()?; + if !re.is_match(document_text.get(path)?) { + return None; + } + Some((view, file_column(rows.get(source)?, path))) + }) + .collect() +} + +/// The column of the cell in `row` that names `path`, so a hit about the file lands where the +/// archivist would look for it. The first column when no cell names it directly. +fn file_column(row: &[SharedString], path: &Path) -> usize { + let names = path + .file_name() + .map(|name| settings::filenames::keys(&name.to_string_lossy())) + .unwrap_or_default(); + row.iter() + .position(|cell| { + settings::filenames::lookup_keys(cell) + .iter() + .any(|key| names.contains(key)) + }) + .unwrap_or(0) +} + /// Rewrite matches in place: `Regex::replace_all` substitutes only the matched spans, so the rest /// of the cell's text survives and a cell with two occurrences gets both. Literal mode wraps the /// replacement in `NoExpand` so a `$` the user typed stays a `$`; regex mode leaves `$1` capture @@ -1379,6 +1549,41 @@ mod tests { assert_eq!(find_matches(&data, &visible, 2, "ap", opts), vec![(0, 0)]); } + /// A hit inside a linked PDF lands on the cell naming the file; a row whose document lacks the + /// query, or that has no document at all, is not reported. + #[test] + fn find_reaches_into_linked_documents() { + let data = rows(&[ + &["Letter to council", "1920_letter.pdf"], + &["Portrait", "1921_portrait.jpg"], + &["Minutes", "1922_minutes.pdf"], + ]); + let paths = vec![ + Some(PathBuf::from("files/1920_letter.pdf")), + Some(PathBuf::from("files/1921_portrait.jpg")), + Some(PathBuf::from("files/1922_minutes.pdf")), + ]; + let text = HashMap::from([ + ( + PathBuf::from("files/1920_letter.pdf"), + "the sawmill on Kitsilano beach".to_string(), + ), + ( + PathBuf::from("files/1922_minutes.pdf"), + "motion carried".to_string(), + ), + ]); + let opts = SearchOpts { + files: true, + ..SearchOpts::default() + }; + assert_eq!( + find_file_matches(&data, &[0, 1, 2], &paths, &text, "SAWMILL", opts), + vec![(0, 1)] + ); + assert!(find_file_matches(&data, &[1, 2], &paths, &text, "sawmill", opts).is_empty()); + } + #[test] fn blank_query_matches_nothing() { let data = rows(&[&["a"]]); @@ -1389,7 +1594,12 @@ mod tests { fn search_toggles_case_word_and_regex() { let data = rows(&[&["Apple pie"], &["pineapple"]]); let all = &[0, 1][..]; - let opt = |case, word, regex| SearchOpts { case, word, regex }; + let opt = |case, word, regex| SearchOpts { + case, + word, + regex, + ..SearchOpts::default() + }; // Match case: "apple" no longer hits the capitalized "Apple pie". assert_eq!( @@ -1463,9 +1673,8 @@ mod tests { fn replace_expands_captures_only_in_regex_mode() { let data = rows(&[&["Smith, Jane"]]); let regex = SearchOpts { - case: false, - word: false, regex: true, + ..SearchOpts::default() }; assert_eq!( replace_edits(&data, &[0], 1, r"(\w+), (\w+)", "$2 $1", regex, None), @@ -1737,6 +1946,52 @@ mod app_tests { }); } + /// Visual results come best first, skip rows the index cannot rate, and stop at the limit. + #[gpui::test] + fn search_rows_reorder_the_view_and_searches_still_see_every_row(cx: &mut TestAppContext) { + let state = table(cx); + cx.update(|cx| { + state.update(cx, |state, _| { + let delegate = state.delegate_mut(); + delegate.set_image_paths(vec![ + Some("0.jpg".into()), + None, + Some("2.jpg".into()), + Some("3.jpg".into()), + ]); + let score = |path: &std::path::Path| match path.to_str() { + Some("0.jpg") => Some(0.2), + Some("2.jpg") => Some(0.9), + _ => None, + }; + let rows = |d: &super::QrateTableDelegate| -> Vec { + d.ranked_rows(score) + .into_iter() + .map(|(_, row, _)| row) + .collect() + }; + assert_eq!(rows(delegate), vec![2, 0]); + + delegate.set_search_rows(Some(vec![2])); + assert_eq!(delegate.visible(), &[2]); + assert_eq!( + rows(delegate), + vec![2, 0], + "a narrowed view still ranks every row" + ); + + delegate.set_search_rows(Some(vec![3, 0])); + assert_eq!(delegate.visible(), &[3, 0], "hits keep their order"); + delegate.remove_rows(&[1]); + assert_eq!( + delegate.visible(), + &[0, 1, 2], + "a delete drops the stale hits" + ); + }); + }); + } + /// A multi-row delete is one undo step, and the deleted rows come back where they were rather /// than all at the first index. #[gpui::test] diff --git a/crates/table/src/lib.rs b/crates/table/src/lib.rs index da46727f..2e82e5b0 100644 --- a/crates/table/src/lib.rs +++ b/crates/table/src/lib.rs @@ -20,6 +20,7 @@ mod note; mod panel; pub mod photos; mod row_index; +mod visual; pub use agent::{AGENT_SOURCE, respond_to_agent, respond_to_agent_async}; pub use delegate::{QrateTableDelegate, Selection, TableChanged}; diff --git a/crates/table/src/note.rs b/crates/table/src/note.rs index 3fba52b2..773a29ba 100644 --- a/crates/table/src/note.rs +++ b/crates/table/src/note.rs @@ -349,13 +349,24 @@ pub fn menu( let (rows, files) = { let delegate = table.read(cx).delegate(); let rows = target_rows(delegate, row); - let files = rows + let files: Vec = rows .iter() .filter_map(|&row| delegate.row_image(row).map(std::path::Path::to_path_buf)) .collect(); (rows, files) }; - selection_items(menu, &rows, files) + match (rows.len(), files.first()) { + (1, Some(file)) => { + let file = file.clone(); + menu.item( + PopupMenuItem::new("Find similar items").on_click(move |_, _, cx| { + crate::visual::find_similar(file.clone(), cx) + }), + ) + .separator() + } + _ => selection_items(menu, &rows, files), + } } None => menu, }; diff --git a/crates/table/src/panel.rs b/crates/table/src/panel.rs index fc13db34..0bfb66e9 100644 --- a/crates/table/src/panel.rs +++ b/crates/table/src/panel.rs @@ -3,10 +3,11 @@ use std::ops::RangeInclusive; use gpui::prelude::FluentBuilder as _; use gpui::*; use gpui_component::{ - ActiveTheme, Disableable as _, IconName, Selectable as _, Sizable as _, + ActiveTheme, Disableable as _, Icon, IconName, Selectable as _, Sizable as _, button::{Button, ButtonVariants as _}, h_flex, input::{Escape, Input, InputEvent, InputState, TextareaState}, + slider::{Slider, SliderEvent, SliderState}, table::{DataTable, TableEvent, TableState}, v_flex, }; @@ -21,7 +22,7 @@ use crate::{ editing::{self, EditState}, floating::float_at, history::Cells, - note, photos, row_index, + note, photos, row_index, visual, }; const COLUMN_LAYOUT_KEY: &str = "table_columns"; @@ -93,6 +94,19 @@ pub struct TablePanel { search_opts: SearchOpts, search_error: bool, _search_sub: Subscription, + /// Linked documents being read for a search that includes them. `Some` while it runs. + reading_documents: Option>, + /// The pending visual query. Replacing it cancels the one before, which debounces typing. + visual_query: Option>, + /// The file a visual search with an empty query ranks against, from "Find similar items". + similar: Option, + _visual_sub: Subscription, + /// The last visual ranking as `(score, source_row, data_col)`, best first, so moving the + /// breadth slider re-cuts it without asking the model again. + ranking: Vec<(f32, usize, usize)>, + /// How far below the best visual hit still counts as a match. + breadth: Entity, + _breadth_sub: Subscription, replace_input: Entity, replace_open: bool, _replace_sub: Subscription, @@ -367,6 +381,38 @@ impl TablePanel { }, ); + visual::init(cx); + // Picks up a "find similar" request from a menu, and re-runs a visual search as the model + // downloads and the index fills in. + let _visual_sub = cx.observe_global_in::(window, |this, window, cx| { + // Read first: `global_mut` notifies this observer again, so calling it unconditionally loops. + if cx.global::().similar.is_some() + && let Some(path) = cx.global_mut::().similar.take() + { + this.similar = Some(path); + this.search_opts.visual = true; + this.search_open = true; + this.search_input.update(cx, |input, cx| { + input.set_value("", window, cx); + input.focus(window, cx); + }); + this.refresh_search(cx); + } else if this.search_open && this.search_opts.visual { + this.refresh_search(cx); + } + }); + + let breadth = cx.new(|_| { + SliderState::new() + .min(*visual::BREADTH_RANGE.start()) + .max(*visual::BREADTH_RANGE.end()) + .step(0.01) + .default_value(visual::BREADTH) + }); + let _breadth_sub = cx.subscribe(&breadth, |this, _, _: &SliderEvent, cx| { + this.show_ranking(cx) + }); + let _replace_sub = cx.subscribe(&replace_input, |this, _input, event: &InputEvent, cx| { if matches!(event, InputEvent::PressEnter { .. }) { this.replace(false, cx); @@ -389,6 +435,13 @@ impl TablePanel { search_opts: SearchOpts::default(), search_error: false, _search_sub, + reading_documents: None, + visual_query: None, + similar: None, + _visual_sub, + ranking: Vec::new(), + breadth, + _breadth_sub, replace_input, replace_open: false, _replace_sub, @@ -431,9 +484,23 @@ impl TablePanel { /// Recompute the find matches from the current query and jump to the first, if any. Called on /// every keystroke in the find bar (the scan is sub-millisecond for qrate's grids). fn refresh_search(&mut self, cx: &mut Context) { + if self.search_opts.visual { + self.search_visual(cx); + return; + } + self.ranking.clear(); let needle = self.search_input.read(cx).value().to_string(); self.search_error = !needle.trim().is_empty() && compile_search(&needle, self.search_opts).is_none(); + // A search that reaches into linked files narrows the view to its hits. + let hits = (self.search_opts.files && !needle.trim().is_empty() && !self.search_error) + .then(|| { + self.state + .read(cx) + .delegate() + .hit_rows(&needle, self.search_opts) + }); + self.set_search_rows(hits, cx); self.search_matches = self .state .read(cx) @@ -441,9 +508,117 @@ impl TablePanel { .search_matches(&needle, self.search_opts); self.search_ix = 0; self.select_current_match(cx); + self.read_documents(cx); + cx.notify(); + } + + /// Rank rows by how well their linked file matches the query, or looks like the file "Find + /// similar items" was raised on when the query is empty. Typing waits for a pause first. + fn search_visual(&mut self, cx: &mut Context) { + let files = self.state.read(cx).delegate().linked_files(); + visual::index(files, cx); + self.search_error = false; + let needle = self.search_input.read(cx).value().trim().to_string(); + let (query, pause) = match (needle.is_empty(), self.similar.clone()) { + (false, _) => ( + visual::Query::Text(needle), + std::time::Duration::from_millis(300), + ), + (true, Some(path)) => (visual::Query::Like(path), std::time::Duration::ZERO), + (true, None) => { + self.visual_query = None; + self.ranking.clear(); + self.show_ranking(cx); + return; + } + }; + self.visual_query = Some(cx.spawn(async move |this, cx| { + cx.background_executor().timer(pause).await; + let Ok(score) = this.update(cx, |_, cx| visual::scorer(query, cx)) else { + return; + }; + let score = score.await; + this.update(cx, |this, cx| { + // No scorer means the model is not loaded yet: show every row until it is. + this.ranking = score + .map(|score| this.state.read(cx).delegate().ranked_rows(score)) + .unwrap_or_default(); + this.show_ranking(cx); + }) + .ok(); + })); + } + + /// Narrow the view to the visual ranking's hits within the breadth slider, best first, and step + /// through them in that order. An empty ranking shows every row. + fn show_ranking(&mut self, cx: &mut Context) { + let breadth = self.breadth.read(cx).value().start(); + let keep = visual::cutoff(self.ranking.iter().map(|hit| hit.0), breadth); + let hits = &self.ranking[..keep]; + let cols: std::collections::HashMap = + hits.iter().map(|&(_, row, col)| (row, col)).collect(); + self.set_search_rows( + (!hits.is_empty()).then(|| hits.iter().map(|hit| hit.1).collect()), + cx, + ); + self.search_matches = self + .state + .read(cx) + .delegate() + .visible() + .iter() + .enumerate() + .filter_map(|(view, source)| cols.get(source).map(|&col| (view, col))) + .collect(); + self.search_ix = 0; + self.select_current_match(cx); cx.notify(); } + fn set_search_rows(&mut self, rows: Option>, cx: &mut Context) { + self.state.update(cx, |state, cx| { + if state.delegate_mut().set_search_rows(rows) { + cx.emit(TableChanged); + cx.notify(); + } + }); + } + + /// Read the text of every linked document the search has not seen yet, off the UI thread, then + /// search again so their hits join the results. + fn read_documents(&mut self, cx: &mut Context) { + if !self.search_opts.files || self.reading_documents.is_some() { + return; + } + let unread = self.state.read(cx).delegate().unread_documents(); + if unread.is_empty() { + return; + } + self.reading_documents = Some(cx.spawn(async move |this, cx| { + let texts = cx + .background_executor() + .spawn(async move { + unread + .into_iter() + .map(|path| { + let text = preview::document_text(&path).unwrap_or_default(); + (path, text) + }) + .collect() + }) + .await; + this.update(cx, |this, cx| { + this.reading_documents = None; + this.state + .update(cx, |state, _| state.delegate_mut().add_document_text(texts)); + if this.search_open { + this.refresh_search(cx); + } + }) + .ok(); + })); + } + fn toggle_opt(&mut self, pick: fn(&mut SearchOpts) -> &mut bool, cx: &mut Context) { let flag = pick(&mut self.search_opts); *flag = !*flag; @@ -452,12 +627,15 @@ impl TablePanel { fn goto_match(&mut self, delta: isize, cx: &mut Context) { // Re-scan first: matches are view indices, and a filter change since last keystroke invalidates them. - let needle = self.search_input.read(cx).value().to_string(); - self.search_matches = self - .state - .read(cx) - .delegate() - .search_matches(&needle, self.search_opts); + // A visual ranking costs a model call, so it keeps the order it was given. + if !self.search_opts.visual { + let needle = self.search_input.read(cx).value().to_string(); + self.search_matches = self + .state + .read(cx) + .delegate() + .search_matches(&needle, self.search_opts); + } let n = self.search_matches.len(); if n == 0 { return; @@ -481,6 +659,9 @@ impl TablePanel { /// whole visible set instead, as a single undo step. Matches are cell-granular, so this /// substitutes every occurrence *within* each affected cell and leaves the rest of its text. fn replace(&mut self, all: bool, cx: &mut Context) { + if self.search_opts.visual { + return; + } let needle = self.search_input.read(cx).value().to_string(); let replacement = self.replace_input.read(cx).value().to_string(); let only = match all { @@ -515,16 +696,42 @@ impl TablePanel { .update(cx, |input, cx| input.focus(window, cx)); self.refresh_search(cx); } else { + self.ranking.clear(); + self.set_search_rows(None, cx); self.focus_handle.focus(window, cx); } cx.notify(); } + /// Open the find bar with its replace row. Replace has nothing to rewrite in a visual search. + pub fn open_replace(&mut self, window: &mut Window, cx: &mut Context) { + if !self.search_opts.visual { + self.replace_open = true; + } + if !self.search_open { + self.toggle_search(window, cx); + } + cx.notify(); + } + + /// Close the find bar if one of its own fields holds focus. `Escape` is bound in the `Input` + /// context, which the cell editor shares, so the caller must propagate when this declines. + pub fn escape_search(&mut self, window: &mut Window, cx: &mut Context) -> bool { + let focused = self.search_input.focus_handle(cx).is_focused(window) + || self.replace_input.focus_handle(cx).is_focused(window); + if focused { + self.dismiss_search(window, cx); + } + focused + } + fn dismiss_search(&mut self, window: &mut Window, cx: &mut Context) { if !self.search_open { return; } self.search_open = false; + self.ranking.clear(); + self.set_search_rows(None, cx); self.focus_handle.focus(window, cx); cx.notify(); } @@ -868,19 +1075,23 @@ impl TablePanel { (hooks.forget_suggestions)(cx); } } -} - -impl Focusable for TablePanel { - fn focus_handle(&self, _: &App) -> FocusHandle { - self.focus_handle.clone() - } -} -impl Render for TablePanel { - fn render(&mut self, window: &mut Window, cx: &mut Context) -> impl IntoElement { - let stripe = settings::effective_bool(crate::TABLE_STRIPES_KEY, cx); + /// The find bar, or nothing while it is closed. Drawn by whichever view is showing, so a search + /// carries on across Table and Gallery. + pub fn render_search_bar(&mut self, cx: &mut Context) -> AnyElement { + if !self.search_open { + return Empty.into_any_element(); + } let query = self.search_input.read(cx).value(); - let count = if self.search_error { + let visual_status = visual::status(cx); + let count = if let Some(status) = self + .search_opts + .visual + .then(|| visual::status_label(&visual_status)) + .flatten() + { + status + } else if self.search_error { SharedString::from("Invalid regex") } else if !self.search_matches.is_empty() { SharedString::from(format!( @@ -889,11 +1100,202 @@ impl Render for TablePanel { self.search_matches.len() )) } else if query.trim().is_empty() { - SharedString::default() + match (&self.similar, self.search_opts.visual) { + (Some(_), true) => SharedString::from("No similar items"), + _ => SharedString::default(), + } + } else if self.reading_documents.is_some() { + SharedString::from("Reading files…") } else { SharedString::from("No results") }; let (border, muted) = (cx.theme().border, cx.theme().muted_foreground); + let opts = self.search_opts; + let toggle = |id: &'static str, + icon: Option, + label: &'static str, + tip: &'static str, + on: bool, + pick: fn(&mut SearchOpts) -> &mut bool| { + Button::new(id) + .ghost() + .small() + .selected(on) + // Text rules have no meaning for a query about what an image shows. + .disabled(opts.visual && id != "search-visual") + .tooltip(tip) + .map(|b| match icon { + Some(icon) => b.icon(icon), + None => b.label(label), + }) + .on_click(cx.listener(move |this, _, _, cx| this.toggle_opt(pick, cx))) + }; + let has_matches = !self.search_matches.is_empty(); + v_flex() + .flex_none() + .gap_1() + .px_2() + .py_2() + .border_b_1() + .border_color(border) + .child( + h_flex() + .gap_1() + .items_center() + .child( + Button::new("replace-toggle") + .ghost() + .small() + .icon(match self.replace_open { + true => IconName::ChevronDown, + false => IconName::ChevronRight, + }) + .disabled(opts.visual) + .tooltip("Toggle replace (Ctrl+H)") + .on_click(cx.listener(|this, _, _, cx| { + this.replace_open = !this.replace_open; + cx.notify(); + })), + ) + .child( + h_flex() + .flex_1() + .items_center() + .gap_1() + .child(div().flex_1().child(Input::new(&self.search_input))) + .child(toggle( + "search-case", + Some(IconName::CaseSensitive.into()), + "", + "Match case", + opts.case, + |o| &mut o.case, + )) + .child(toggle( + "search-word", + None, + "W", + "Match whole word", + opts.word, + |o| &mut o.word, + )) + .child(toggle( + "search-regex", + None, + ".*", + "Use regular expression", + opts.regex, + |o| &mut o.regex, + )) + .child(toggle( + "search-files", + Some(IconName::BookOpen.into()), + "", + "Include text inside linked files", + opts.files, + |o| &mut o.files, + )) + .child(toggle( + "search-visual", + Some(Icon::empty().path("icons/image.svg")), + "", + "Search by what images show", + opts.visual, + |o| &mut o.visual, + )), + ) + .child( + div() + .min_w(px(64.)) + .text_xs() + .text_color(muted) + .child(count), + ) + .when(opts.visual, |bar| match visual_status { + visual::Status::Missing | visual::Status::Failed(_) => bar.child( + Button::new("visual-install") + .small() + .label(visual::download_label()) + .on_click(|_, _, cx| visual::install(cx)), + ), + _ => bar.child( + div() + .id("visual-breadth") + .w(px(80.)) + .tooltip(|window, cx| { + gpui_component::tooltip::Tooltip::new("Fewer or more matches") + .build(window, cx) + }) + .child(Slider::new(&self.breadth)), + ), + }) + .child( + Button::new("search-prev") + .icon(IconName::ChevronUp) + .ghost() + .small() + .tooltip("Previous match (Shift+Enter)") + .on_click(cx.listener(|this, _, _, cx| this.goto_match(-1, cx))), + ) + .child( + Button::new("search-next") + .icon(IconName::ChevronDown) + .ghost() + .small() + .tooltip("Next match (Enter)") + .on_click(cx.listener(|this, _, _, cx| this.goto_match(1, cx))), + ) + .child( + Button::new("search-close") + .icon(IconName::Close) + .ghost() + .small() + .tooltip("Close find (Esc)") + .on_click( + cx.listener(|this, _, window, cx| this.dismiss_search(window, cx)), + ), + ), + ) + .when(self.replace_open && !opts.visual, |bar| { + bar.child( + h_flex() + .gap_1() + .items_center() + .pl_7() + .child(div().flex_1().child(Input::new(&self.replace_input))) + .child( + Button::new("replace-one") + .ghost() + .small() + .label("Replace") + .disabled(!has_matches) + .tooltip("Replace in this cell (Enter)") + .on_click(cx.listener(|this, _, _, cx| this.replace(false, cx))), + ) + .child( + Button::new("replace-all") + .ghost() + .small() + .label("All") + .disabled(!has_matches) + .tooltip("Replace every match in the visible rows") + .on_click(cx.listener(|this, _, _, cx| this.replace(true, cx))), + ), + ) + }) + .into_any_element() + } +} + +impl Focusable for TablePanel { + fn focus_handle(&self, _: &App) -> FocusHandle { + self.focus_handle.clone() + } +} + +impl Render for TablePanel { + fn render(&mut self, window: &mut Window, cx: &mut Context) -> impl IntoElement { + let stripe = settings::effective_bool(crate::TABLE_STRIPES_KEY, cx); v_flex() .size_full() @@ -902,28 +1304,10 @@ impl Render for TablePanel { .id("table-panel") .role(Role::Group) .aria_label("Table") - .on_action(cx.listener(|this, _: &Search, window, cx| this.toggle_search(window, cx))) - .on_action(cx.listener(|this, _: &Replace, window, cx| { - this.replace_open = true; - if !this.search_open { - this.toggle_search(window, cx); - } - cx.notify(); - })) - // The find input propagates Escape (it doesn't consume it), so dismiss the bar here. - // - // Only when the bar's own fields hold the key, though. `Escape` is bound in the `Input` - // context, which the *cell* editor is in too, and this listener is an ancestor of both — - // so without the check, Escape in a cell editor either closed the find bar behind it or, - // with the bar already shut, was swallowed by a no-op handler and did nothing at all. - // An action stops propagating by default, so declining it has to be explicit. + // Search, Replace and the find bar's own Escape are handled by `ViewsPanel`, which draws + // the bar above every view. An action stops propagating by default, so declining an + // Escape that is not ours has to be explicit. .on_action(cx.listener(|this, _: &Escape, window, cx| { - if this.search_input.focus_handle(cx).is_focused(window) - || this.replace_input.focus_handle(cx).is_focused(window) - { - this.dismiss_search(window, cx); - return; - } // Like a cell edit, a note owns the blur that Escape is about to cause. Drop that // ownership first or `_note_sub` will mistake the blur for a commit. if this.cancel_note_edit(window, cx) { @@ -991,156 +1375,6 @@ impl Render for TablePanel { ) .p_2() .gap_2() - // Find bar renders here, not in `title_suffix`: the parent `TabPanel` never observes us to redraw it. - .when(self.search_open, |this| { - let opts = self.search_opts; - let toggle = |id: &'static str, - icon: Option, - label: &'static str, - tip: &'static str, - on: bool, - pick: fn(&mut SearchOpts) -> &mut bool| { - Button::new(id) - .ghost() - .small() - .selected(on) - .tooltip(tip) - .map(|b| match icon { - Some(icon) => b.icon(icon), - None => b.label(label), - }) - .on_click(cx.listener(move |this, _, _, cx| this.toggle_opt(pick, cx))) - }; - let has_matches = !self.search_matches.is_empty(); - this.child( - v_flex() - .flex_none() - .gap_1() - .pb_2() - .border_b_1() - .border_color(border) - .child( - h_flex() - .gap_1() - .items_center() - .child( - Button::new("replace-toggle") - .ghost() - .small() - .icon(match self.replace_open { - true => IconName::ChevronDown, - false => IconName::ChevronRight, - }) - .tooltip("Toggle replace (Ctrl+H)") - .on_click(cx.listener(|this, _, _, cx| { - this.replace_open = !this.replace_open; - cx.notify(); - })), - ) - .child( - h_flex() - .flex_1() - .items_center() - .gap_1() - .child(div().flex_1().child(Input::new(&self.search_input))) - .child(toggle( - "search-case", - Some(IconName::CaseSensitive), - "", - "Match case", - opts.case, - |o| &mut o.case, - )) - .child(toggle( - "search-word", - None, - "W", - "Match whole word", - opts.word, - |o| &mut o.word, - )) - .child(toggle( - "search-regex", - None, - ".*", - "Use regular expression", - opts.regex, - |o| &mut o.regex, - )), - ) - .child( - div() - .min_w(px(64.)) - .text_xs() - .text_color(muted) - .child(count), - ) - .child( - Button::new("search-prev") - .icon(IconName::ChevronUp) - .ghost() - .small() - .tooltip("Previous match (Shift+Enter)") - .on_click( - cx.listener(|this, _, _, cx| this.goto_match(-1, cx)), - ), - ) - .child( - Button::new("search-next") - .icon(IconName::ChevronDown) - .ghost() - .small() - .tooltip("Next match (Enter)") - .on_click( - cx.listener(|this, _, _, cx| this.goto_match(1, cx)), - ), - ) - .child( - Button::new("search-close") - .icon(IconName::Close) - .ghost() - .small() - .tooltip("Close find (Esc)") - .on_click(cx.listener(|this, _, window, cx| { - this.dismiss_search(window, cx) - })), - ), - ) - .when(self.replace_open, |bar| { - bar.child( - h_flex() - .gap_1() - .items_center() - .pl_7() - .child(div().flex_1().child(Input::new(&self.replace_input))) - .child( - Button::new("replace-one") - .ghost() - .small() - .label("Replace") - .disabled(!has_matches) - .tooltip("Replace in this cell (Enter)") - .on_click(cx.listener(|this, _, _, cx| { - this.replace(false, cx) - })), - ) - .child( - Button::new("replace-all") - .ghost() - .small() - .label("All") - .disabled(!has_matches) - .tooltip("Replace every match in the visible rows") - .on_click( - cx.listener(|this, _, _, cx| { - this.replace(true, cx) - }), - ), - ), - ) - }), - ) - }) .child( div() .flex_1() diff --git a/crates/table/src/visual.rs b/crates/table/src/visual.rs new file mode 100644 index 00000000..cb99ca85 --- /dev/null +++ b/crates/table/src/visual.rs @@ -0,0 +1,419 @@ +//! Visual search for the open table: installing the model, keeping its index current for the rows' +//! linked files, and turning a query into ranked rows. +//! +//! Shared across windows as a global, because the model holds 600 MB and one copy is plenty. + +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use gpui::{App, Global, SharedString, Task}; +use settings::project::{CurrentProject, FILES_FOLDER_KEY, VisualEntry}; +use visual_search::Clip; + +/// Files embedded per model call. Larger batches barely help on a CPU and delay the progress count. +const BATCH: usize = 16; + +/// The most rows a visual search keeps, however broad the search. +const MAX_RESULTS: usize = 500; + +/// The breadth a new search starts at, and the range its slider allows. +pub(crate) const BREADTH: f32 = 0.15; +pub(crate) const BREADTH_RANGE: std::ops::RangeInclusive = 0.02..=0.4; + +/// How many of `scores` (best first) to keep: those within `breadth` of the best, proportionally. +/// +/// ponytail: a proportional gap from the best hit, so text queries (best around 0.3) and find +/// similar (best 1.0) share one slider. Tuned on one collection; calibrate per model if it misjudges. +pub(crate) fn cutoff(scores: impl IntoIterator, breadth: f32) -> usize { + let mut scores = scores.into_iter(); + let Some(best) = scores.next() else { + return 0; + }; + let floor = best * (1.0 - breadth); + (1 + scores.take_while(|&score| score >= floor).count()).min(MAX_RESULTS) +} + +#[derive(Clone, PartialEq)] +pub(crate) enum Status { + Missing, + Downloading(u64), + Indexing { done: usize, total: usize }, + Ready, + Failed(SharedString), +} + +/// Each linked file's vector, with the file length it was embedded at. +/// +/// ponytail: staleness is judged on length alone, so a copied project keeps its vectors when file +/// times change. Store a content hash if same-length edits turn out to matter. +type Index = HashMap)>; + +pub(crate) struct Visual { + pub status: Status, + clip: Option>, + index: Arc>, + /// The `.qrate` file `index` belongs to. Another project starts from its own stored index. + project: Option, + /// Files the indexing job has already been handed this session, so asking again is free. + seen: HashSet, + job: Option>, + /// A "find similar" request waiting for the search bar to pick it up. + pub similar: Option, +} + +impl Global for Visual {} + +fn model_dir() -> Option { + Some( + settings::data_dir()? + .join("models") + .join(visual_search::MODEL_DIR), + ) +} + +pub(crate) fn init(cx: &mut App) { + if cx.has_global::() { + return; + } + let installed = model_dir().is_some_and(|dir| visual_search::installed(&dir)); + cx.set_global(Visual { + status: if installed { + Status::Ready + } else { + Status::Missing + }, + clip: None, + index: Arc::default(), + project: None, + seen: HashSet::new(), + job: None, + similar: None, + }); +} + +/// For writes only: every call notifies the global's observers, which re-run the search bar. +fn state(cx: &mut App) -> &mut Visual { + cx.global_mut::() +} + +pub(crate) fn status(cx: &App) -> Status { + cx.global::().status.clone() +} + +/// What the search bar says instead of a match count while visual search cannot answer yet. +pub(crate) fn status_label(status: &Status) -> Option { + match status { + Status::Missing => Some("Model not installed".into()), + Status::Downloading(done) => Some( + format!( + "Downloading {}%", + done * 100 / visual_search::DOWNLOAD_SIZE.max(1) + ) + .into(), + ), + Status::Indexing { done, total } => Some(format!("Indexing {done} / {total}").into()), + Status::Ready => None, + Status::Failed(reason) => Some(reason.clone()), + } +} + +pub(crate) fn download_label() -> SharedString { + format!( + "Download model ({} MB)", + visual_search::DOWNLOAD_SIZE / 1_000_000 + ) + .into() +} + +/// Fetch the model, reporting progress into [`Status::Downloading`]. +pub(crate) fn install(cx: &mut App) { + let Some(dir) = model_dir() else { + return; + }; + if matches!(status(cx), Status::Downloading(_)) { + return; + } + state(cx).status = Status::Downloading(0); + let progress = Arc::new(AtomicU64::new(0)); + let finished = Arc::new(AtomicBool::new(false)); + let download = cx.background_executor().spawn({ + let (progress, finished) = (progress.clone(), finished.clone()); + async move { + let result = + visual_search::download(&dir, &|bytes| progress.store(bytes, Ordering::Relaxed)); + finished.store(true, Ordering::Relaxed); + result + } + }); + let job = cx.spawn(async move |cx| { + while !finished.load(Ordering::Relaxed) { + cx.background_executor() + .timer(Duration::from_millis(250)) + .await; + cx.update(|cx| { + state(cx).status = Status::Downloading(progress.load(Ordering::Relaxed)) + }); + } + let result = download.await; + cx.update(|cx| { + let visual = state(cx); + visual.job = None; + visual.status = match result { + Ok(()) => Status::Ready, + Err(err) => { + log::error!("could not download the visual search model: {err:#}"); + Status::Failed("Download failed".into()) + } + }; + }); + }); + state(cx).job = Some(job); +} + +/// Embed any of `paths` the index does not already hold at their current length, storing the +/// vectors in the open project. Returns at once; progress shows in [`Status::Indexing`]. +pub(crate) fn index(paths: Vec, cx: &mut App) { + let project = cx.try_global::().map(|project| { + let folder = project + .data + .values + .get(FILES_FOLDER_KEY) + .map(|v| PathBuf::from(v.text().to_string())) + .unwrap_or_default(); + (project.file.clone(), folder) + }); + let visual = cx.global::(); + let Some(dir) = model_dir() else { + return; + }; + if visual.status != Status::Ready || visual.job.is_some() { + return; + } + let switched = visual.project != project.as_ref().map(|(file, _)| file.clone()); + let fresh: Vec = paths + .into_iter() + .filter(|path| switched || !visual.seen.contains(path)) + .collect(); + if fresh.is_empty() { + return; + } + let clip = visual.clip.clone(); + let visual = state(cx); + if switched { + visual.project = project.as_ref().map(|(file, _)| file.clone()); + visual.seen.clear(); + visual.index = Arc::default(); + } + visual.seen.extend(fresh.iter().cloned()); + let index = visual.index.clone(); + let job = cx.spawn(async move |cx| { + let prepared = cx + .background_executor() + .spawn({ + let (index, project) = (index.clone(), project.clone()); + async move { + let clip = match clip { + Some(clip) => clip, + None => Arc::new(Clip::load(&dir).map_err(|err| format!("{err:#}"))?), + }; + if switched && let Some((file, folder)) = &project { + match settings::project::read_visual_index(file, visual_search::MODEL) { + Ok(stored) => lock(&index).extend( + stored + .into_iter() + .map(|(path, len, vector)| (folder.join(path), (len, vector))), + ), + Err(err) => log::warn!( + "could not read the project's visual search index, rebuilding it: {err:#}" + ), + } + } + // Stat outside the lock: searches rank rows on the UI thread through this index. + let stale: Vec<(PathBuf, u64)> = fresh + .into_iter() + .filter_map(|path| { + let len = std::fs::metadata(&path).ok()?.len(); + let kept = lock(&index).get(&path).map(|(kept, _)| *kept); + (kept != Some(len)).then_some((path, len)) + }) + .collect(); + Ok::<_, String>((clip, stale)) + } + }) + .await; + let (clip, stale) = match prepared { + Ok(prepared) => prepared, + Err(err) => { + log::error!("could not load the visual search model: {err}"); + cx.update(|cx| { + let visual = state(cx); + visual.job = None; + visual.status = Status::Failed("Model failed to load".into()); + }); + return; + } + }; + cx.update(|cx| state(cx).clip = Some(clip.clone())); + + let total = stale.len(); + for (at, batch) in stale.chunks(BATCH).enumerate() { + cx.update(|cx| { + state(cx).status = Status::Indexing { + done: at * BATCH, + total, + } + }); + let (clip, index, project, batch) = + (clip.clone(), index.clone(), project.clone(), batch.to_vec()); + cx.background_executor() + .spawn(async move { + let embedded = embed_batch(&clip, batch); + if let Some((file, folder)) = &project { + let stored: Vec = embedded + .iter() + .map(|(path, len, vector)| (stored_path(path, folder), *len, vector.clone())) + .collect(); + if let Err(err) = + settings::project::write_visual_index(file, visual_search::MODEL, &stored) + { + log::warn!("could not save visual search vectors to the project, they will be rebuilt: {err:#}"); + } + } + lock(&index).extend( + embedded + .into_iter() + .map(|(path, len, vector)| (path, (len, vector))), + ); + }) + .await; + } + if total > 0 { + log::info!("visual search indexed {total} files"); + } + cx.update(|cx| { + let visual = state(cx); + visual.job = None; + visual.status = Status::Ready; + }); + }); + state(cx).job = Some(job); +} + +fn lock(index: &Mutex) -> std::sync::MutexGuard<'_, Index> { + index.lock().unwrap_or_else(|err| err.into_inner()) +} + +/// `path` relative to the files folder with `/` separators, so a project moved with its files, or +/// to another platform, still finds its vectors. Absolute when the file lives elsewhere. +fn stored_path(path: &Path, folder: &Path) -> String { + match path.strip_prefix(folder) { + Ok(relative) if !folder.as_os_str().is_empty() => relative + .components() + .map(|part| part.as_os_str().to_string_lossy()) + .collect::>() + .join("/"), + _ => path.to_string_lossy().into_owned(), + } +} + +fn embed_batch(clip: &Clip, batch: Vec<(PathBuf, u64)>) -> Vec<(PathBuf, u64, Vec)> { + let (files, images): (Vec<_>, Vec<_>) = batch + .into_iter() + .filter_map(|(path, len)| { + let pixels = preview::thumbnail_pixels(&path, preview::CARD, 0)?; + Some(((path, len), pixels)) + }) + .unzip(); + if images.is_empty() { + return Vec::new(); + } + match clip.embed_images(&images) { + Ok(vectors) => files + .into_iter() + .zip(vectors) + .map(|((path, len), vector)| (path, len, vector)) + .collect(), + Err(err) => { + log::warn!("visual search skipped {} files: {err:#}", files.len()); + Vec::new() + } + } +} + +/// What the query should be compared against: a description, or another file's own vector. +pub(crate) enum Query { + Text(String), + Like(PathBuf), +} + +/// A scoring function over files for [`crate::delegate::QrateTableDelegate::ranked_rows`], or +/// `None` when the model is not loaded or the query cannot be embedded. +pub(crate) fn scorer( + query: Query, + cx: &App, +) -> Task Option + use<>>> { + let visual = cx.global::(); + let (clip, index) = (visual.clip.clone(), visual.index.clone()); + cx.background_executor().spawn(async move { + let vector = match query { + Query::Text(text) => clip? + .embed_text(&text) + .map_err(|err| log::warn!("could not embed a visual search query: {err:#}")) + .ok()?, + Query::Like(path) => lock(&index).get(&path)?.1.clone(), + }; + Some(move |path: &Path| { + let index = lock(&index); + Some(visual_search::similarity(&vector, &index.get(path)?.1)) + }) + }) +} + +/// Ask the search bar to show the files that look most like `path`. +pub(crate) fn find_similar(path: PathBuf, cx: &mut App) { + state(cx).similar = Some(path); +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use super::{cutoff, stored_path}; + + #[test] + fn stored_paths_are_relative_to_the_files_folder() { + let folder = Path::new("/archive/files"); + assert_eq!( + stored_path(&folder.join("box 1").join("a.jpg"), folder), + "box 1/a.jpg" + ); + assert_eq!( + stored_path(Path::new("/elsewhere/b.jpg"), folder), + "/elsewhere/b.jpg" + ); + assert_eq!( + stored_path(Path::new("/x/c.jpg"), Path::new("")), + "/x/c.jpg" + ); + } + + #[test] + fn cutoff_keeps_hits_near_the_best_one() { + assert_eq!(cutoff([], 0.15), 0); + assert_eq!(cutoff([0.30, 0.28, 0.26, 0.20], 0.15), 3); + assert_eq!( + cutoff([0.30, 0.28, 0.26, 0.20], 0.02), + 1, + "the best hit always stays" + ); + assert_eq!( + cutoff([1.0, 0.9, 0.5], 0.15), + 2, + "find similar starts from itself" + ); + assert_eq!(cutoff(vec![0.3; 900], 0.4), 500, "capped"); + } +} diff --git a/crates/visual-search/Cargo.toml b/crates/visual-search/Cargo.toml new file mode 100644 index 00000000..a0ba7649 --- /dev/null +++ b/crates/visual-search/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "visual-search" +version.workspace = true +edition.workspace = true + +[dependencies] +anyhow.workspace = true +log.workspace = true +serde_json.workspace = true +# Pure Rust, so the model runs on every platform CI builds. Held at 0.9: from 0.10 candle-core +# pulls in `tokenizers` and its C oniguruma build, which nothing here calls. +candle-core = "0.9.2" +candle-nn = "0.9.2" +candle-transformers = "0.9.2" +image = { version = "0.25", default-features = false } +regex = "1" +reqwest = { version = "0.12", features = ["blocking"] } +sha2 = "0.10" + +[dev-dependencies] +dirs.workspace = true diff --git a/crates/visual-search/src/lib.rs b/crates/visual-search/src/lib.rs new file mode 100644 index 00000000..c6975a61 --- /dev/null +++ b/crates/visual-search/src/lib.rs @@ -0,0 +1,203 @@ +//! Searching a collection by what its images show. +//! +//! CLIP turns an image and a sentence into vectors in the same space, so "a sawmill by the water" +//! lands near photographs of one. Every linked file's thumbnail is embedded once and stored in the +//! project; a query is embedded as text, or taken from another image for "find similar", and rows +//! are ranked by how close their vectors are. +//! +//! The model runs on the CPU through candle, which is pure Rust. Its weights are not part of the +//! install: [`download`] fetches a pinned revision and checks it against the published checksum. + +mod tokenizer; + +use std::fs; +use std::io::{Read as _, Write as _}; +use std::path::Path; + +use anyhow::{Context as _, Result, ensure}; +use candle_core::{DType, Device, Tensor}; +use candle_transformers::models::clip::{ClipConfig, ClipModel, div_l2_norm}; +use image::{DynamicImage, RgbaImage, imageops::FilterType}; +use sha2::{Digest as _, Sha256}; + +/// Which weights an index was built with. Vectors from different models are not comparable, so an +/// index recorded under another name is discarded rather than searched. +pub const MODEL: &str = "openai/clip-vit-base-patch32@b33cedf"; + +/// Folder name for the weights under qrate's models directory. +pub const MODEL_DIR: &str = "clip-vit-base-patch32"; + +const REVISION: &str = "b33cedfd0df4e43b8238760678fcc89e1a0d38b3"; + +struct File { + name: &'static str, + size: u64, + sha256: &'static str, +} + +const FILES: [File; 2] = [ + File { + name: "tokenizer.json", + size: 2_224_041, + sha256: "b556ac8c99757ffb677208af34bc8c6721572114111a6e0aaf5fa69ff0b8d842", + }, + File { + name: "model.safetensors", + size: 605_157_884, + sha256: "99d28a652e6ec46629ab7047a0ac82c69b1fe11e0ce672c43af65d3a9a3fc05d", + }, +]; + +/// Bytes [`download`] fetches in total, for the install prompt. +pub const DOWNLOAD_SIZE: u64 = FILES[0].size + FILES[1].size; + +/// Whether the weights are in `dir`. Checks sizes, not checksums: [`download`] only renames a file +/// into place after its checksum matched. +pub fn installed(dir: &Path) -> bool { + FILES + .iter() + .all(|file| fs::metadata(dir.join(file.name)).is_ok_and(|meta| meta.len() == file.size)) +} + +/// Fetch the pinned weights into `dir`, calling `progress` with bytes done so far. A file is written +/// beside its final name and moved into place only once its SHA-256 matches, so an interrupted or +/// tampered download never looks installed. +pub fn download(dir: &Path, progress: &dyn Fn(u64)) -> Result<()> { + fs::create_dir_all(dir).with_context(|| format!("could not create {}", dir.display()))?; + // No overall timeout: the weights are 600 MB and a slow connection is not a failure. + let client = reqwest::blocking::Client::builder().timeout(None).build()?; + let mut done = 0; + for file in &FILES { + let target = dir.join(file.name); + if fs::metadata(&target).is_ok_and(|meta| meta.len() == file.size) { + done += file.size; + progress(done); + continue; + } + let url = format!( + "https://huggingface.co/openai/clip-vit-base-patch32/resolve/{REVISION}/{}", + file.name + ); + let mut response = client.get(&url).send()?.error_for_status()?; + let partial = target.with_extension("part"); + let mut out = fs::File::create(&partial)?; + let mut hasher = Sha256::new(); + let mut buffer = vec![0; 1 << 16]; + loop { + let read = response.read(&mut buffer)?; + if read == 0 { + break; + } + out.write_all(&buffer[..read])?; + hasher.update(&buffer[..read]); + done += read as u64; + progress(done); + } + drop(out); + let digest = format!("{:x}", hasher.finalize()); + if digest != file.sha256 { + let _ = fs::remove_file(&partial); + } + ensure!( + digest == file.sha256, + "the downloaded {} did not match its published checksum", + file.name + ); + fs::rename(&partial, &target)?; + } + Ok(()) +} + +/// The loaded model. Holds about 600 MB, memory-mapped from the weights file. +pub struct Clip { + model: ClipModel, + tokenizer: tokenizer::Tokenizer, +} + +/// The normalisation CLIP was trained with, per RGB channel. +const MEAN: [f32; 3] = [0.481_454_66, 0.457_827_5, 0.408_210_73]; +const STD: [f32; 3] = [0.268_629_54, 0.261_302_6, 0.275_777_1]; + +impl Clip { + pub fn load(dir: &Path) -> Result { + let tokenizer = tokenizer::Tokenizer::load(&dir.join("tokenizer.json"))?; + // SAFETY: the weights file is only ever replaced by rename, never modified in place, so the + // mapping cannot change underneath the model. + let weights = unsafe { + candle_nn::VarBuilder::from_mmaped_safetensors( + &[dir.join("model.safetensors")], + DType::F32, + &Device::Cpu, + )? + }; + let model = ClipModel::new(weights, &ClipConfig::vit_base_patch32())?; + Ok(Self { model, tokenizer }) + } + + /// One unit-length vector per image, in order. + pub fn embed_images(&self, images: &[RgbaImage]) -> Result>> { + let size = ClipConfig::vit_base_patch32().image_size; + let mut pixels = Vec::with_capacity(images.len() * 3 * size * size); + for image in images { + // Fill then centre-crop, as CLIP's own preprocessing does, so nothing is squashed. + let rgb = DynamicImage::ImageRgba8(image.clone()) + .resize_to_fill(size as u32, size as u32, FilterType::CatmullRom) + .to_rgb8(); + for channel in 0..3 { + pixels.extend( + rgb.pixels().map(|px| { + (f32::from(px.0[channel]) / 255.0 - MEAN[channel]) / STD[channel] + }), + ); + } + } + let batch = Tensor::from_vec(pixels, (images.len(), 3, size, size), &Device::Cpu)?; + Ok(div_l2_norm(&self.model.get_image_features(&batch)?)?.to_vec2()?) + } + + /// A unit-length vector for a description of what an image shows. + pub fn embed_text(&self, text: &str) -> Result> { + let ids = Tensor::new(self.tokenizer.encode(text), &Device::Cpu)?.unsqueeze(0)?; + let features = div_l2_norm(&self.model.get_text_features(&ids)?)?; + Ok(features.squeeze(0)?.to_vec1()?) + } +} + +/// Cosine similarity of two unit-length vectors. +pub fn similarity(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b).map(|(a, b)| a * b).sum() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Runs against the real weights when they are installed. Without them this is the skip path, + /// not a pass — `download` into the models folder first. + #[test] + fn a_description_finds_the_image_it_describes() { + let Some(dir) = dirs::data_local_dir().map(|d| d.join("qrate/models").join(MODEL_DIR)) + else { + return; + }; + if !installed(&dir) { + eprintln!("skipping the model check: CLIP weights are not installed"); + return; + } + let clip = Clip::load(&dir).unwrap(); + let red = RgbaImage::from_pixel(64, 64, image::Rgba([220, 20, 20, 255])); + let blue = RgbaImage::from_pixel(64, 64, image::Rgba([20, 40, 220, 255])); + let images = clip.embed_images(&[red, blue]).unwrap(); + assert_eq!(images[0].len(), 512); + + let query = clip.embed_text("a plain red square").unwrap(); + assert!( + (similarity(&query, &query) - 1.0).abs() < 1e-3, + "unit length" + ); + assert!( + similarity(&query, &images[0]) > similarity(&query, &images[1]), + "red ranks above blue" + ); + } +} diff --git a/crates/visual-search/src/tokenizer.rs b/crates/visual-search/src/tokenizer.rs new file mode 100644 index 00000000..bb248f7d --- /dev/null +++ b/crates/visual-search/src/tokenizer.rs @@ -0,0 +1,135 @@ +//! CLIP's byte-level BPE, read from the model's own `tokenizer.json`. +//! +//! ponytail: skips the NFC normalisation step, which only changes text with decomposed accents. +//! Add `unicode-normalization` if queries in such text rank oddly. + +use std::collections::HashMap; +use std::path::Path; + +use anyhow::{Context as _, Result}; +use regex::Regex; + +const START: u32 = 49406; +const END: u32 = 49407; +/// The text encoder's position embeddings stop here, start and end tokens included. +const CONTEXT: usize = 77; + +pub struct Tokenizer { + vocab: HashMap, + ranks: HashMap<(String, String), usize>, + split: Regex, + bytes: [char; 256], +} + +impl Tokenizer { + pub fn load(file: &Path) -> Result { + let json: serde_json::Value = serde_json::from_slice(&std::fs::read(file)?)?; + let model = &json["model"]; + let vocab = serde_json::from_value(model["vocab"].clone()).context("tokenizer vocab")?; + let ranks = model["merges"] + .as_array() + .context("tokenizer merges")? + .iter() + .filter_map(|merge| merge.as_str()?.split_once(' ')) + .enumerate() + .map(|(rank, (a, b))| ((a.to_owned(), b.to_owned()), rank)) + .collect(); + Ok(Self { + vocab, + ranks, + split: Regex::new(r"'s|'t|'re|'ve|'m|'ll|'d|\p{L}+|\p{N}|[^\s\p{L}\p{N}]+")?, + bytes: byte_chars(), + }) + } + + pub fn encode(&self, text: &str) -> Vec { + let text = text + .split_whitespace() + .collect::>() + .join(" ") + .to_lowercase(); + let mut ids = vec![START]; + for word in self.split.find_iter(&text) { + let symbols = word + .as_str() + .bytes() + .map(|b| self.bytes[b as usize].to_string()); + ids.extend( + self.merge(symbols.collect()) + .iter() + .map(|symbol| self.vocab.get(symbol).copied().unwrap_or(END)), + ); + } + ids.truncate(CONTEXT - 1); + ids.push(END); + ids + } + + /// Apply merges lowest rank first until none apply, marking the word's end the way the + /// vocabulary does. + fn merge(&self, mut symbols: Vec) -> Vec { + if let Some(last) = symbols.last_mut() { + last.push_str(""); + } + loop { + let best = symbols + .windows(2) + .enumerate() + .filter_map(|(at, pair)| { + let rank = self.ranks.get(&(pair[0].clone(), pair[1].clone()))?; + Some((*rank, at)) + }) + .min(); + let Some((_, at)) = best else { + return symbols; + }; + let right = symbols.remove(at + 1); + symbols[at].push_str(&right); + } + } +} + +/// GPT-2's reversible byte-to-character table: printable bytes stand for themselves, the rest are +/// shifted past 255 so every byte has a visible, vocabulary-safe character. +fn byte_chars() -> [char; 256] { + let mut table = ['\0'; 256]; + let mut shifted = 0; + for byte in 0..=255u8 { + table[byte as usize] = match byte { + b'!'..=b'~' | 0xA1..=0xAC | 0xAE..=0xFF => char::from(byte), + _ => { + shifted += 1; + char::from_u32(255 + shifted).unwrap_or('\0') + } + }; + } + table +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unprintable_bytes_move_past_latin_one() { + let table = byte_chars(); + assert_eq!(table[b'a' as usize], 'a'); + assert_eq!(table[b' ' as usize], '\u{120}', "GPT-2 writes a space as Ġ"); + } + + #[test] + fn encodes_like_the_reference_tokenizer() { + let Some(file) = dirs::data_local_dir() + .map(|d| d.join("qrate/models/clip-vit-base-patch32/tokenizer.json")) + .filter(|file| file.is_file()) + else { + eprintln!("skipping the tokenizer check: CLIP weights are not installed"); + return; + }; + let tokenizer = Tokenizer::load(&file).unwrap(); + assert_eq!( + tokenizer.encode("A photo of a CAT"), + vec![START, 320, 1125, 539, 320, 2368, END] + ); + } +} diff --git a/crates/workspace/src/viewer/mod.rs b/crates/workspace/src/viewer/mod.rs index c4ce9aee..4732eac5 100644 --- a/crates/workspace/src/viewer/mod.rs +++ b/crates/workspace/src/viewer/mod.rs @@ -87,7 +87,28 @@ pub fn open_viewer(path: PathBuf, scope: Scope, window: &mut Window, cx: &mut Ap .try_global::() .and_then(|active| active.return_focus.clone()) .or_else(|| window.focused(cx)); + let table = cx + .try_global::() + .and_then(|handle| handle.0.upgrade()); let viewer = cx.new(|cx| Viewer { + _follow: table.map(|table| { + cx.subscribe_in( + &table, + window, + |this: &mut Viewer, table, _: &table::TableChanged, window, cx| { + let delegate = table.read(cx).delegate(); + let file = match delegate.selection() { + Some(table::Selection::Cell { row, .. } | table::Selection::Row(row)) => { + previewable(delegate, row) + } + _ => None, + }; + if let Some(file) = file.filter(|file| *file != this.path) { + open_viewer(file, this.scope, window, cx); + } + }, + ) + }), transport: Transport::new(path.clone(), cx), path, details, @@ -131,6 +152,67 @@ pub fn open_viewer(path: PathBuf, scope: Scope, window: &mut Window, cx: &mut Ap }); } +/// Select the next row, by `delta`, in the view's order that has something to preview. During a +/// search the view is its hits, so this steps through the results. The open viewer follows the +/// selection to that row's file. +fn step_row(delta: isize, cx: &mut App) { + let Some(state) = cx + .try_global::() + .and_then(|handle| handle.0.upgrade()) + else { + return; + }; + let target = { + let delegate = state.read(cx).delegate(); + let visible = delegate.visible(); + let from = match delegate.selection() { + Some(table::Selection::Cell { row, .. } | table::Selection::Row(row)) => { + delegate.view_row(row) + } + _ => None, + }; + from.and_then(|from| { + next_row(from, delta, visible.len(), |view| { + previewable(delegate, visible[view]).is_some() + }) + }) + }; + if let Some(view) = target { + state.update(cx, |state, cx| { + state.delegate_mut().clear_selection(); + state.set_selected_row(view, cx); + }); + } +} + +/// The file `row` links to, if the viewer can show it. +fn previewable(delegate: &table::QrateTableDelegate, row: usize) -> Option { + delegate + .row_image(row) + .filter(|file| preview::can_preview(file)) + .map(|file| file.to_path_buf()) +} + +/// The nearest view index past `from` in `delta`'s direction that `viewable` accepts, if any. +fn next_row( + from: usize, + delta: isize, + len: usize, + viewable: impl Fn(usize) -> bool, +) -> Option { + let step = delta.signum(); + let mut view = from as isize; + loop { + view += step; + if step == 0 || view < 0 || view >= len as isize { + return None; + } + if viewable(view as usize) { + return Some(view as usize); + } + } +} + pub fn close_viewer(window: &mut Window, cx: &mut App) { // Without this the recording plays on over an empty screen, with nothing left to stop it. preview::playback::stop(cx); @@ -183,6 +265,9 @@ pub struct Viewer { /// carries the drag handle, the sizing and the propagation rules, none of which are ours to /// reinvent. split: Entity, + /// Swaps in the selected row's file when the selection moves, from the find bar, the arrows or + /// anywhere else. + _follow: Option, } impl Viewer { @@ -456,6 +541,9 @@ impl Render for Viewer { this.set_zoom(1.0); cx.notify(); } + "up" | "down" if reading => { + step_row(if ev.keystroke.key == "up" { -1 } else { 1 }, cx); + } _ => {} } })) @@ -704,6 +792,35 @@ impl Render for Viewer { ) }, ) + // Rows, not pages: the neighbouring files in the view's order, which during a search are + // the neighbouring results. + .child( + div() + .absolute() + .bottom_4() + .right_4() + .flex() + .gap_1() + .p_1() + .rounded(cx.theme().radius) + .bg(pill) + .child( + Button::new("previous-row") + .icon(IconName::ArrowLeft) + .ghost() + .small() + .tooltip("Previous row (↑)") + .on_click(|_, _, cx| step_row(-1, cx)), + ) + .child( + Button::new("next-row") + .icon(IconName::ArrowRight) + .ghost() + .small() + .tooltip("Next row (↓)") + .on_click(|_, _, cx| step_row(1, cx)), + ), + ) .child( div() .absolute() @@ -781,7 +898,17 @@ mod tests { VisualTestContext, Window, div, }; - use crate::viewer::{Scope, close_viewer, open_viewer, viewer_in}; + use crate::viewer::{Scope, close_viewer, next_row, open_viewer, viewer_in}; + + #[test] + fn stepping_rows_skips_what_cannot_be_previewed_and_stops_at_the_ends() { + let viewable = |view: usize| view != 2; + assert_eq!(next_row(1, 1, 5, viewable), Some(3)); + assert_eq!(next_row(3, -1, 5, viewable), Some(1)); + assert_eq!(next_row(4, 1, 5, viewable), None); + assert_eq!(next_row(0, -1, 5, viewable), None); + assert_eq!(next_row(3, 1, 4, viewable), None); + } struct FocusProbe(FocusHandle); diff --git a/crates/workspace/src/views/gallery.rs b/crates/workspace/src/views/gallery.rs index 851f08e8..574fdfb4 100644 --- a/crates/workspace/src/views/gallery.rs +++ b/crates/workspace/src/views/gallery.rs @@ -27,6 +27,7 @@ pub(super) fn render( state: Option>>, width: Pixels, cols: usize, + scroll: &UniformListScrollHandle, focus: &FocusHandle, cx: &mut App, ) -> AnyElement { @@ -65,6 +66,7 @@ pub(super) fn render( .collect() }, ) + .track_scroll(scroll) .size_full() .p_2() .into_any_element() diff --git a/crates/workspace/src/views/mod.rs b/crates/workspace/src/views/mod.rs index ea37c618..05d67bce 100644 --- a/crates/workspace/src/views/mod.rs +++ b/crates/workspace/src/views/mod.rs @@ -19,11 +19,12 @@ use gpui_component::{ button::Button, dock::{BasePanel, DockArea, Panel, PanelEvent}, h_flex, + input::Escape, slider::{Slider, SliderEvent, SliderState}, tab::{Tab, TabBar}, v_flex, }; -use table::{QrateTableDelegate, TableChanged, TablePanel, TableStateHandle}; +use table::{QrateTableDelegate, Replace, Search, TableChanged, TablePanel, TableStateHandle}; use crate::ViewerScope; @@ -136,6 +137,12 @@ pub struct ViewsPanel { thumb: Entity, /// Persists the count as it is dragged, and repaints the cards to match. _thumb_sub: Subscription, + /// Repaints the find bar, which this panel draws on the table's behalf, as the search moves. + _table_sub: Subscription, + gallery_scroll: UniformListScrollHandle, + /// The row the gallery last scrolled to, so it follows the selection (stepping through search + /// hits) without pulling back a card the user has scrolled away from. + gallery_followed: Option, } impl ViewsPanel { @@ -169,7 +176,7 @@ impl ViewsPanel { }); let mut this = Self { focus_handle: cx.focus_handle(), - table, + table: table.clone(), dock_area, view: ViewMode::parse(&settings::effective_text(VIEW_MODE_KEY, cx)), body_width: px(0.), @@ -182,6 +189,9 @@ impl ViewsPanel { _viewer_sub: cx.observe_global::(|_, cx| cx.notify()), thumb, _thumb_sub, + _table_sub: cx.observe(&table, |_, _, cx| cx.notify()), + gallery_scroll: UniformListScrollHandle::new(), + gallery_followed: None, }; this.bind(cx); cx.set_global(ActiveViewsPanel(cx.entity().downgrade())); @@ -337,11 +347,8 @@ impl Panel for ViewsPanel { } /// Rendered by `TabPanel::render_toolbar` immediately left of the ⋯ menu, forced to - /// `.xsmall().ghost()` by the library. Only the grid has anything to find in. + /// `.xsmall().ghost()` by the library. Every view shows the same rows, so every view can find in them. fn toolbar_buttons(&mut self, _w: &mut Window, _cx: &mut Context) -> Option> { - if self.view != ViewMode::Table { - return None; - } let table = self.table.downgrade(); Some(vec![ Button::new("table-search") @@ -366,6 +373,27 @@ impl Render for ViewsPanel { .id("views-panel") .role(Role::Group) .aria_label("Views") + // The find bar lives above whichever view is showing, so its keys are handled here too. + .on_action(cx.listener(|this, _: &Search, window, cx| { + this.table + .update(cx, |table, cx| table.toggle_search(window, cx)) + })) + .on_action(cx.listener(|this, _: &Replace, window, cx| { + this.table + .update(cx, |table, cx| table.open_replace(window, cx)) + })) + .on_action(cx.listener(|this, _: &Escape, window, cx| { + if !this + .table + .update(cx, |table, cx| table.escape_search(window, cx)) + { + cx.propagate(); + } + })) + .child( + self.table + .update(cx, |table, cx| table.render_search_bar(cx)), + ) .child( div() .flex_1() @@ -391,13 +419,35 @@ impl Render for ViewsPanel { // chained-builder types meeting in one `match` overflows rustc's stack. .child(match self.view { ViewMode::Table => self.table.clone().into_any_element(), - ViewMode::Gallery => gallery::render( - self.state.as_ref().and_then(WeakEntity::upgrade), - self.body_width, - self.thumb.read(cx).value().start().round() as usize, - &self.focus_handle, - cx, - ), + ViewMode::Gallery => { + let state = self.state.as_ref().and_then(WeakEntity::upgrade); + let cols = self.thumb.read(cx).value().start().round() as usize; + let cursor = state.as_ref().and_then(|state| { + let delegate = state.read(cx).delegate(); + match delegate.selection()? { + table::Selection::Cell { row, .. } + | table::Selection::Row(row) => delegate.view_row(row), + table::Selection::Column(_) => None, + } + }); + if cursor != self.gallery_followed { + self.gallery_followed = cursor; + if let Some(view) = cursor { + self.gallery_scroll.scroll_to_item( + view / cols.max(1), + ScrollStrategy::Nearest, + ); + } + } + gallery::render( + state, + self.body_width, + cols, + &self.gallery_scroll, + &self.focus_handle, + cx, + ) + } }) .children(crate::viewer::viewer_in(ViewerScope::Centre, cx)), ) diff --git a/docs/files-and-photos.md b/docs/files-and-photos.md index 890df8d1..539316dc 100644 --- a/docs/files-and-photos.md +++ b/docs/files-and-photos.md @@ -24,6 +24,10 @@ previews images, documents, audio, and video directly. Click the preview to open fullscreen, where you can zoom, pan, page through a multi-page document, and search inside it. +The arrows in the bottom-right corner of the fullscreen view move to the file of the +previous or next row, in the order the view shows them. During a search that means the +previous or next result. + PDF previews need PDFium and video frame previews need ffmpeg. qrate looks for both beside its own executable, then on your system `PATH`. Without them, qrate shows a file-type icon instead of a preview, and the rest of the app works as normal. diff --git a/docs/grid.md b/docs/grid.md index 7c8ed763..da0c71cd 100644 --- a/docs/grid.md +++ b/docs/grid.md @@ -21,7 +21,14 @@ it. A frozen row or column stays in view while you scroll. ## Search and replace Open search with **Ctrl+F**. Search moves through matching cells in the grid. Open replace -with **Ctrl+H** to replace one match or all matches at once. +with **Ctrl+H** to replace one match or all matches at once. The search bar belongs to +whichever view is showing, so a search carries on when you switch between the grid and the +gallery. + +Two toggles in the search bar look beyond the cells, and both narrow the view to the rows +they match. The book searches the text inside linked PDFs. The picture searches what the +linked images show, without using their metadata at all. See +[Visual search](visual-search.md). ## Filtering diff --git a/docs/index.md b/docs/index.md index 7f95bb45..178102a6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,6 +8,7 @@ checks the data against validators, and exports it to the formats other systems - [Projects](projects.md) — create, import, and open a `.qrate` project - [The grid](grid.md) — edit cells, search, filter, and undo - [Files and photos](files-and-photos.md) — link records to files, and view them +- [Visual search](visual-search.md) — find records by what their pictures show - [Diagnostics](diagnostics.md) — the Problems panel, spelling, and fixes - [Columns](columns.md) — column types, authority lists, and per-column settings - [Export and Google Sheets](export-and-sync.md) — CSV, JSON-LD, CSL-JSON, ZIP, and Sheets sync diff --git a/docs/visual-search.md b/docs/visual-search.md new file mode 100644 index 00000000..e9332131 --- /dev/null +++ b/docs/visual-search.md @@ -0,0 +1,111 @@ +# Visual search + +Visual search finds records by what their pictures show, rather than by what their +metadata says. Search for `a wooden bridge` and qrate ranks the collection's images by how +well each one matches that description, even when no cell holds any of those words. + +It also answers the other direction: pick a record and ask for the items that look like +it. + +Everything runs on your own computer. No image and no search text leaves it. + +## What it can search + +Visual search reads the same thumbnails the gallery shows, so it covers every file type +qrate can preview: + +- Photographs and other images. +- PDFs and documents, by their first page. +- Video, by its poster frame. +- Audio, by its cover art, if it has any. + +A row with no linked file cannot be found this way. See +[Files and photos](files-and-photos.md) for how rows link to files. + +## Installing the model + +The first time you turn visual search on, qrate offers a **Download model** button in the +search bar. The download is about 605 MB and happens once per computer, not once per +project. qrate checks the download against a published checksum before installing it. + +The model is stored beside qrate's other data, never inside a project, and it is shared by +every project you open. + +## Searching + +1. Open search with **Ctrl+F**. +2. Turn on the picture toggle in the search bar. +3. Type what you are looking for, in plain words: `a group of people outside a building`. + +While a visual search is running: + +- **The view shows only the results**, best match first. This applies to both the grid and + the gallery, and the search stays as you switch between them. +- **The slider** beside the result count widens or narrows the results. Drag it left for + only the closest matches and right to include weaker ones. Results are scored relative to + the best match, so the slider means the same thing for any search. +- **The arrows** step through the results in order. If a file is open fullscreen, stepping + changes the file it shows. +- **Replace is switched off**, along with match case, whole word and regular expressions. + None of them mean anything for a picture. + +Clear the search box or close the search bar to bring every row back. + +### Writing a good query + +- **Describe the picture, not the record.** `a horse-drawn cart on a dirt road` works; + `accession 1994.12` does not. +- **Plain descriptions beat keywords.** `two people shaking hands` beats `handshake photo + formal`. +- **Expect near misses.** The model ranks every image, so weaker matches follow the good + ones. Tighten the slider rather than reading further down. +- **It reads the picture, not the text in it.** Searching for words written on a document + is a job for the linked-file text search, below. + +## Find similar items + +Right-click a row, or a gallery tile, and choose **Find similar items**. qrate ranks every +other file by how much it looks like that one, and narrows the view to the closest. + +This is how you find the rest of a photo shoot, the other copies of a form, or the pages +that came from the same album. + +## Indexing + +Before a file can be found, qrate reads its thumbnail once and stores what the model made +of it. This is the indexing step, and the search bar shows its progress. + +- **It takes about a tenth of a second per file**, so a few thousand files take a few + minutes the first time. +- **The results are stored in the project file**, so the next time you open the project + there is nothing to redo. +- **A file is read again only if it changes.** Copying a project and its files to another + computer keeps the index. +- **It runs in the background.** You can keep working, though the app may feel slower while + it works through a large collection for the first time. + +If you change the files folder or add new files, only the new files are read. + +## Searching inside linked files + +The book toggle in the search bar, beside the picture one, searches the *text inside* +linked PDFs rather than their appearance. It is off by default. Turn it on and an ordinary +search also matches words in a linked document, narrowing the view to the rows whose files +contain them. + +- The text of each document is read once and kept, so later searches are immediate. +- A hit lands on the cell that names the file. +- Replace does not change text inside a file. It only rewrites cells. +- A scan with no text layer has nothing to search. qrate does not yet read text from + scanned images. + +## Limits + +- **It is a ranking, not a verdict.** The model has no idea what your collection is about, + and it can be confidently wrong. Treat a result as a suggestion to check. +- **It reads one page or frame per file.** Page 40 of a PDF is not searched. +- **It does not read handwriting or printed text.** +- **Terms are English.** The model was trained on English descriptions, so queries in other + languages match poorly. +- **Nothing is written to your data.** Visual search only finds and orders rows. It never + fills a cell in.