From c07fb358425081b378ded227698a30c4cd659e6b Mon Sep 17 00:00:00 2001 From: flupkede Date: Tue, 15 Sep 2026 16:18:47 +0200 Subject: [PATCH] [worker] fase D2: tantivy 0.22 -> 0.26 with graceful FTS reset - TopDocs::with_limit now needs .order_by_score() (collector API change), applied at the three search call sites - tantivy cannot open index files written by an older major; an unreadable FTS index is now wiped + recreated instead of failing the DB open. FTS is derived data: BM25 rebuilds on next (re)index, vector search unaffected. Warning logged. Regression test: corrupt meta.json. - merged origin/develop (fastembed 6 CHANGELOG repair + entries) Validation: fmt, clippy -D warnings, cargo test --lib --bins green. --- CHANGELOG.md | 2 + Cargo.lock | 216 +++++++++++++++++++++------------------ Cargo.toml | 2 +- src/fts/tantivy_store.rs | 91 ++++++++++++++--- 4 files changed, 195 insertions(+), 116 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ed3197f..ce71facb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,8 @@ finalized in place with a date — no renaming/migration step needed. - **Embedding stack majors: fastembed 6.1, hf-hub 1.0, ndarray 0.17.** Zero call-site changes — the codesearch embedder sits on fastembed's `ModelType`/`InitOptions` surface, which 6.1 kept stable. Scope note: fastembed 6.1 still resolves `image 0.25`, so the weezl 0.1.12/moxcms 0.8.1 advisories (Aikido 30640676/37515815) remain open until fastembed adopts image 0.26+; likewise rand 0.9.5 stays via hf-hub 0.5/tokenizers. +- **tantivy 0.22 → 0.26 with graceful FTS reset.** The collector API changed so `TopDocs::with_limit` needs `.order_by_score()` at the three call sites. Because tantivy cannot open index files written by an older major, an unreadable FTS index is now wiped and recreated as a fresh empty index instead of failing the whole DB open — the FTS index is derived data, BM25 results rebuild on the next (re)index, and a warning is logged pointing at `codesearch index`. Vector search and all non-FTS paths are unaffected; pinned by a regression test that feeds the store a corrupt `meta.json`. + ## [1.3.19] ### Changed diff --git a/Cargo.lock b/Cargo.lock index d5c73ccd..02255a52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1234,6 +1234,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "datasketches" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" + [[package]] name = "der" version = "0.8.2" @@ -1387,9 +1393,9 @@ dependencies = [ [[package]] name = "downcast-rs" -version = "1.2.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" [[package]] name = "doxygen-rs" @@ -1485,6 +1491,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + [[package]] name = "errno" version = "0.3.14" @@ -1652,12 +1669,12 @@ dependencies = [ [[package]] name = "fs4" -version = "0.8.4" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" dependencies = [ - "rustix 0.38.44", - "windows-sys 0.52.0", + "rustix 1.1.4", + "windows-sys 0.59.0", ] [[package]] @@ -2478,18 +2495,6 @@ dependencies = [ "syn 3.0.5", ] -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "interpolate_name" version = "0.2.4" @@ -2501,6 +2506,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + [[package]] name = "ipnet" version = "2.12.2" @@ -2533,15 +2547,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -2803,16 +2808,19 @@ dependencies = [ ] [[package]] -name = "lru-slab" -version = "0.1.3" +name = "lru" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" +dependencies = [ + "hashbrown 0.16.1", +] [[package]] -name = "lz4_flex" -version = "0.11.6" +name = "lru-slab" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" [[package]] name = "lz4_flex" @@ -2882,11 +2890,10 @@ dependencies = [ [[package]] name = "measure_time" -version = "0.8.3" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbefd235b0aadd181626f281e1d684e116972988c14c264e42069d5e8a5775cc" +checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e" dependencies = [ - "instant", "log", ] @@ -3264,7 +3271,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", - "libm", ] [[package]] @@ -3465,9 +3471,9 @@ dependencies = [ [[package]] name = "ownedbytes" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558" +checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e" dependencies = [ "stable_deref_trait", ] @@ -3821,7 +3827,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.3", + "rustc-hash", "rustls", "socket2", "thiserror 2.0.20", @@ -3843,7 +3849,7 @@ dependencies = [ "rand 0.10.2", "rand_pcg", "ring", - "rustc-hash 2.1.3", + "rustc-hash", "rustls", "rustls-pki-types", "slab", @@ -3894,8 +3900,6 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ - "libc", - "rand_chacha 0.3.1", "rand_core 0.6.4", ] @@ -3905,7 +3909,7 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ - "rand_chacha 0.9.0", + "rand_chacha", "rand_core 0.9.5", ] @@ -3920,16 +3924,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - [[package]] name = "rand_chacha" version = "0.9.0" @@ -3945,9 +3939,6 @@ name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] [[package]] name = "rand_core" @@ -3964,16 +3955,6 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" -[[package]] -name = "rand_distr" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" -dependencies = [ - "num-traits", - "rand 0.8.8", -] - [[package]] name = "rand_pcg" version = "0.10.2" @@ -3996,7 +3977,7 @@ dependencies = [ "indoc", "instability", "itertools 0.13.0", - "lru", + "lru 0.12.5", "paste", "strum", "unicode-segmentation", @@ -4032,7 +4013,7 @@ dependencies = [ "paste", "profiling", "rand 0.9.5", - "rand_chacha 0.9.0", + "rand_chacha", "simd_helpers", "thiserror 2.0.20", "v_frame", @@ -4383,12 +4364,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.3" @@ -4858,9 +4833,9 @@ checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" [[package]] name = "sketches-ddsketch" -version = "0.2.2" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" +checksum = "513c3f5f732bfd6fbb187619c2dfe9d2f25f1a2976f01d575f0fd329d565df56" dependencies = [ "serde", ] @@ -5085,37 +5060,38 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tantivy" -version = "0.22.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96599ea6fccd844fc833fed21d2eecac2e6a7c1afd9e044057391d78b1feb141" +checksum = "861facfabd71044968f364837f9a083b56464ba5a59079f88706ee5c451ca069" dependencies = [ "aho-corasick", "arc-swap", "base64 0.22.1", "bitpacking", + "bon", "byteorder", "census", "crc32fast", "crossbeam-channel", + "datasketches", "downcast-rs", "fastdivide", "fnv", "fs4", "htmlescape", - "itertools 0.12.1", + "itertools 0.14.0", "levenshtein_automata", "log", - "lru", - "lz4_flex 0.11.6", + "lru 0.16.4", + "lz4_flex", "measure_time", "memmap2", - "num_cpus", "once_cell", "oneshot", "rayon", "regex", "rust-stemmers", - "rustc-hash 1.1.0", + "rustc-hash", "serde", "serde_json", "sketches-ddsketch", @@ -5128,30 +5104,31 @@ dependencies = [ "tantivy-stacker", "tantivy-tokenizer-api", "tempfile", - "thiserror 1.0.69", + "thiserror 2.0.20", "time", + "typetag", "uuid", "winapi", ] [[package]] name = "tantivy-bitpacker" -version = "0.6.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df" +checksum = "4fed3d674429bcd2de5d0a6d1aa5495fed8afd9c5ecce993019caf7615f53fa4" dependencies = [ "bitpacking", ] [[package]] name = "tantivy-columnar" -version = "0.3.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e" +checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc" dependencies = [ "downcast-rs", "fastdivide", - "itertools 0.12.1", + "itertools 0.14.0", "serde", "tantivy-bitpacker", "tantivy-common", @@ -5161,9 +5138,9 @@ dependencies = [ [[package]] name = "tantivy-common" -version = "0.7.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4" +checksum = "bbf10915aa75da3c3b0d58b58853d2e889efbaf32d4982a4c3715dde6bba23e5" dependencies = [ "async-trait", "byteorder", @@ -5185,19 +5162,25 @@ dependencies = [ [[package]] name = "tantivy-query-grammar" -version = "0.22.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82" +checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82" dependencies = [ + "fnv", "nom 7.1.3", + "ordered-float", + "serde", + "serde_json", ] [[package]] name = "tantivy-sstable" -version = "0.3.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e" +checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606" dependencies = [ + "futures-util", + "itertools 0.14.0", "tantivy-bitpacker", "tantivy-common", "tantivy-fst", @@ -5206,20 +5189,19 @@ dependencies = [ [[package]] name = "tantivy-stacker" -version = "0.3.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8" +checksum = "6cbb051742da9d53ca9e8fff43a9b10e319338b24e2c0e15d0372df19ffeb951" dependencies = [ "murmurhash32", - "rand_distr", "tantivy-common", ] [[package]] name = "tantivy-tokenizer-api" -version = "0.3.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04" +checksum = "eac258c2c6390673f2685813afeeafcb8c4e0ee7de8dd3fc46838dcc37263f98" dependencies = [ "serde", ] @@ -5835,12 +5817,42 @@ version = "2.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "typetag" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "typewit" version = "1.15.2" @@ -6672,7 +6684,7 @@ dependencies = [ "getrandom 0.4.3", "heapify", "itertools 0.14.0", - "lz4_flex 0.13.1", + "lz4_flex", "more-asserts", "rand 0.10.2", "regex", diff --git a/Cargo.toml b/Cargo.toml index 42961d8c..96fdb2c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,7 +67,7 @@ fs2 = "0.4" # Cross-platform file locking moka = { version = "0.12", features = ["sync"] } # Search & Ranking -tantivy = "0.22" +tantivy = "0.26" regex = "1.12" # Server diff --git a/src/fts/tantivy_store.rs b/src/fts/tantivy_store.rs index 459980f5..45e7eba4 100644 --- a/src/fts/tantivy_store.rs +++ b/src/fts/tantivy_store.rs @@ -177,16 +177,17 @@ impl FtsStore { /// Open or create index with retry logic for Windows file locking issues fn open_or_create_index_with_retry(fts_path: &Path, schema: &Schema) -> Result { - let max_retries = 3; + const MAX_RETRIES: usize = 3; let mut last_error: Option = None; - for attempt in 0..max_retries { + for attempt in 0..MAX_RETRIES { if attempt > 0 { // Wait before retry (exponential backoff) std::thread::sleep(std::time::Duration::from_millis(100 * (1 << attempt))); } - let result: Result = if fts_path.join("meta.json").exists() { + let opened_existing = fts_path.join("meta.json").exists(); + let result: Result = if opened_existing { Index::open_in_dir(fts_path).map_err(|e| e.to_string()) } else { MmapDirectory::open(fts_path) @@ -202,18 +203,53 @@ impl FtsStore { Err(e) => { last_error = Some(e); // On Windows, try to clear lock files if permission denied - if attempt < max_retries - 1 { + if attempt < MAX_RETRIES - 1 { Self::try_clear_lock_files(fts_path); } } } } - Err(anyhow!( - "Failed to open FTS index after {} retries: {}", - max_retries, - last_error.unwrap_or_default() - )) + // Last resort: a pre-upgrade FTS index written by an older tantivy + // major cannot be opened by this one (format break), which would + // brick the whole DB for existing users. The FTS index is derived + // data — fully rebuildable via reindex — so wipe it and create a + // fresh empty index instead of failing. BM25 results stay empty + // until the next (re)index; vector search is unaffected. + if let Err(wipe_err) = Self::wipe_fts_dir(fts_path) { + return Err(anyhow!( + "Failed to open FTS index after {MAX_RETRIES} retries: {} (wipe also failed: {wipe_err})", + last_error.unwrap_or_default() + )); + } + tracing::warn!( + "FTS index at {} was unreadable by this codesearch version and has been reset; \ + run 'codesearch index' to rebuild it", + fts_path.display() + ); + MmapDirectory::open(fts_path) + .map_err(|e| e.to_string()) + .and_then(|dir| { + Index::create(dir, schema.clone(), IndexSettings::default()) + .map_err(|e| e.to_string()) + }) + .map_err(|e| { + anyhow!( + "Failed to open FTS index after {} retries: {} (fresh create also failed: {})", + MAX_RETRIES, + last_error.unwrap_or_default(), + e + ) + }) + } + + /// Remove the FTS index directory contents so a fresh index can be created. + fn wipe_fts_dir(fts_path: &Path) -> std::io::Result<()> { + if !fts_path.exists() { + return Ok(()); + } + std::fs::remove_dir_all(fts_path)?; + std::fs::create_dir_all(fts_path) } /// Create writer with retry logic for Windows file locking issues @@ -502,7 +538,8 @@ impl FtsStore { }; // Execute search - let top_docs = searcher.search(&parsed_query, &TopDocs::with_limit(limit))?; + let top_docs = + searcher.search(&parsed_query, &TopDocs::with_limit(limit).order_by_score())?; self.collect_fts_results(top_docs) } @@ -560,7 +597,7 @@ impl FtsStore { BooleanQuery::union(vec![Box::new(boosted_sig), Box::new(content_query)]) }; - let top_docs = searcher.search(&combined, &TopDocs::with_limit(limit))?; + let top_docs = searcher.search(&combined, &TopDocs::with_limit(limit).order_by_score())?; self.collect_fts_results(top_docs) } @@ -578,7 +615,7 @@ impl FtsStore { let query = RegexQuery::from_pattern(pattern, self.content_field) .map_err(|e| anyhow!("Invalid regex pattern '{}': {}", pattern, e))?; - let top_docs = searcher.search(&query, &TopDocs::with_limit(limit))?; + let top_docs = searcher.search(&query, &TopDocs::with_limit(limit).order_by_score())?; self.collect_fts_results(top_docs) } @@ -628,7 +665,7 @@ impl FtsStore { Box::new(PhraseQuery::new(terms)) }; - let top_docs = searcher.search(&query, &TopDocs::with_limit(limit))?; + let top_docs = searcher.search(&query, &TopDocs::with_limit(limit).order_by_score())?; self.collect_fts_results(top_docs) } @@ -845,4 +882,32 @@ mod tests { Ok(()) } + + /// An index written by an incompatible tantivy major must not brick the + /// store: the unreadable index is wiped and a fresh empty one created + /// (BM25 rebuilds on the next index run). + #[test] + fn unreadable_index_is_wiped_and_store_recovers() -> Result<()> { + let tmp = tempfile::tempdir()?; + let fts_dir = tmp.path().join("fts"); + std::fs::create_dir_all(&fts_dir)?; + // A meta.json no tantivy version can parse simulates an index from + // an incompatible tantivy major. + std::fs::write(fts_dir.join("meta.json"), "{ not valid tantivy metadata")?; + + let mut store = FtsStore::new(tmp.path())?; + + // Fresh store: opens and searches empty. + let results = store.search("anything", 10, None)?; + assert!(results.is_empty()); + + // And accepts writes after the wipe. + store.add_chunk(1, "recovery probe content", "probe.rs", None, "block")?; + store.commit()?; + let results = store.search("recovery probe", 10, None)?; + assert_eq!(results.len(), 1); + assert_eq!(results[0].chunk_id, 1); + + Ok(()) + } }