feat: add IVF-SQ L2 distance range search

core/README.md

@@ -22,14 +22,15 @@
 `paimon-vindex-core` contains the Rust implementations and seek-based readers
 for IVF-FLAT, IVF-SQ, IVF-PQ, IVF-RQ, and DiskANN.
 
-The Rust reader supports distance range search for IVF-FLAT and IVF-RQ with
+The Rust reader supports distance range search for IVF-FLAT, IVF-RQ, and IVF-SQ with
 squared L2, using `DistanceBand`, `VectorRangeSearchParams`, and CSR
-`RangeSearchResult` buffers. Both families support single and batch queries,
+`RangeSearchResult` buffers. All three families support single and batch queries,
 with or without a serialized Roaring allow-list, and a fixed positive `nprobe`.
 IVF-FLAT tests exact distances; IVF-RQ tests its one-bit or full multi-bit
-estimated distances. Results are uncapped and unordered. Probing every list
-removes the IVF coverage gap, but not IVF-RQ's quantization error. The range
-path does not change top-K search or the v1 storage format.
+estimated distances; IVF-SQ tests scalar-quantized estimates. Results are uncapped
+and unordered. Probing every list removes the IVF coverage gap, but not the
+quantization error of IVF-RQ or IVF-SQ. The range path does not change top-K
+search or the v1 storage format.
 
 See the [range search guide](../docs/range-search.html) for membership,
 validation, filtering, and statistics. C/JNI range bindings are not included.
@@ -41,6 +42,27 @@ abstractions. It does not incorporate source code from Microsoft's
 The implementation supports L2, inner-product, and cosine search with the same
 lower-is-better distance semantics as the IVF indexes.
 
+## IVF-SQ range search
+
+Use `DistanceBand` and `VectorRangeSearchParams` with `range_search`,
+`range_search_batch`, or their `*_with_roaring_filter` variants. Bands are
+half-open `[lower, upper)` in squared-L2 units; results use `RangeSearchResult`
+CSR buffers without sorting, padding, or a top-K cap.
+
+IVF-SQ uses the same blocked SIMD estimator as top-K, including the stored
+per-list residual bounds. Raising `nprobe` visits more lists but does not remove
+quantization error: even at `nprobe == nlist`, membership can differ from the
+original vectors' distances at either boundary. Prefer IVF-FLAT when exact
+membership is required. There is no original-vector reranking or top-K fallback.
+
+Batch queries share list reads, reuse the existing partition cache, and evaluate
+the Roaring allow-list once per list row using query-local one-bit-per-row masks.
+Large lists stream in bounded chunks; scan scratch is reused, and a finite upper cut
+allows entire SQ blocks to stop after their partial distances reach that cut.
+Result memory still grows with the number of hits. Cache hits are excluded from
+`call_stats().list_reads()`. Range support does not extend to other metrics,
+IVF-PQ, DiskANN, or language bindings in this change.
+
 The crate ships its [normative v1 storage-format specification](STORAGE_FORMAT.md)
 and byte-exact fixtures. Project documentation, language bindings, and
 contribution guidance live in the

core/src/collect.rs

@@ -26,8 +26,8 @@
 //! `scan_codes_range` taking a `RangeQueryResult&`; one collector serves both
 //! here, so there is a single kernel rather than a pair to keep in step.
 //!
-//! `ivfflat_io::ReaderTopKHeap` and [`RangeCollector`] are the two
-//! implementations.
+//! `ivfflat_io::ReaderTopKHeap`, `topk::TopKHeap`, and [`RangeCollector`]
+//! implement the collection policies.
 
 use std::io;
 
@@ -64,7 +64,8 @@ pub(crate) trait Collector {
     fn cutoff(&self) -> f32;
 
     /// Delivers one row, with the value the family's scan computed for it. For
-    /// IVF-Flat that value is an exact distance; for IVF-RQ it is an estimate.
+    /// IVF-Flat that value is an exact distance; for IVF-RQ and IVF-SQ it is an
+    /// estimate.
     ///
     /// Fallible because a collector may own a resource the scan cannot see: the
     /// oversized-list path streams chunks through a callback, and without a
@@ -130,6 +131,12 @@ impl RangeCollector {
     pub(crate) fn into_rows(self) -> Vec<(i64, f32)> {
         self.rows
     }
+
+    pub(crate) fn merge(&mut self, mut other: Self) {
+        self.scanned += other.scanned;
+        self.early_abandoned += other.early_abandoned;
+        self.rows.append(&mut other.rows);
+    }
 }
 
 impl Collector for RangeCollector {

core/src/index.rs

@@ -1866,9 +1866,9 @@ impl<R: SeekRead> VectorIndexReader<R> {
     }
 
     /// Distance range search. For the contract see
-    /// [`IVFFlatIndexReader::range_search`] and
-    /// [`IVFRQIndexReader::range_search`]. IVF-RQ membership uses estimated
-    /// distances rather than distances to the original vectors.
+    /// [`IVFFlatIndexReader::range_search`] (exact distances),
+    /// [`IVFRQIndexReader::range_search`] (RQ estimates), and
+    /// [`IVFSQIndexReader::range_search`] (SQ estimates). Only L2 is supported.
     ///
     /// The empty-band short-circuit lives **inside each family's reader**, so a
     /// family that cannot do range search at all still fails loud for every
@@ -1883,15 +1883,16 @@ impl<R: SeekRead> VectorIndexReader<R> {
         match self {
             Self::IvfFlat(reader) => reader.range_search(query, params),
             Self::IvfRq(reader) => reader.range_search(query, params),
-            Self::IvfSq(_) => Err(range_unsupported("ivf_sq")),
+            Self::IvfSq(reader) => reader.range_search(query, params),
             Self::IvfPq(_) => Err(range_unsupported("ivf_pq")),
             Self::DiskAnn(_) => Err(range_unsupported("diskann")),
         }
     }
 
     /// Range search restricted to a serialized Roaring allow-list. For the
-    /// contract see [`IVFFlatIndexReader::range_search_with_roaring_filter`]
-    /// and [`IVFRQIndexReader::range_search`].
+    /// contract see [`IVFFlatIndexReader::range_search_with_roaring_filter`],
+    /// [`IVFRQIndexReader::range_search`], and
+    /// [`IVFSQIndexReader::range_search_with_roaring_filter`].
     pub fn range_search_with_roaring_filter(
         &mut self,
         query: &[f32],
@@ -1907,15 +1908,14 @@ impl<R: SeekRead> VectorIndexReader<R> {
         match self {
             Self::IvfFlat(reader) => reader.range_search_with_filter(query, params, Some(&filter)),
             Self::IvfRq(reader) => reader.range_search_with_filter(query, params, Some(&filter)),
-            Self::IvfSq(_) => Err(range_unsupported("ivf_sq")),
+            Self::IvfSq(reader) => reader.range_search_with_filter(query, params, Some(&filter)),
             Self::IvfPq(_) => Err(range_unsupported("ivf_pq")),
             Self::DiskAnn(_) => Err(range_unsupported("diskann")),
         }
     }
 
     /// Batched distance range search. For the contract see
-    /// [`IVFFlatIndexReader::range_search`] and
-    /// [`IVFRQIndexReader::range_search`].
+    /// [`Self::range_search`].
     pub fn range_search_batch(
         &mut self,
         queries: &[f32],
@@ -1927,7 +1927,7 @@ impl<R: SeekRead> VectorIndexReader<R> {
         match self {
             Self::IvfFlat(reader) => reader.range_search_batch(queries, query_count, params),
             Self::IvfRq(reader) => reader.range_search_batch(queries, query_count, params),
-            Self::IvfSq(_) => Err(range_unsupported("ivf_sq")),
+            Self::IvfSq(reader) => reader.range_search_batch(queries, query_count, params),
             Self::IvfPq(_) => Err(range_unsupported("ivf_pq")),
             Self::DiskAnn(_) => Err(range_unsupported("diskann")),
         }
@@ -1951,7 +1951,9 @@ impl<R: SeekRead> VectorIndexReader<R> {
             Self::IvfRq(reader) => {
                 reader.range_search_batch_with_filter(queries, query_count, params, Some(&filter))
             }
-            Self::IvfSq(_) => Err(range_unsupported("ivf_sq")),
+            Self::IvfSq(reader) => {
+                reader.range_search_batch_with_filter(queries, query_count, params, Some(&filter))
+            }
             Self::IvfPq(_) => Err(range_unsupported("ivf_pq")),
             Self::DiskAnn(_) => Err(range_unsupported("diskann")),
         }