diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 6dc8af48a6e..fada4784ef0 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -85,7 +85,7 @@ jobs: linux-build: runs-on: "ubuntu-24.04-8x" - timeout-minutes: 60 + timeout-minutes: 75 env: # Need up-to-date compilers for kernels CC: clang diff --git a/docs/src/guide/migration.md b/docs/src/guide/migration.md index 804e6dcf3e5..3222326bea6 100644 --- a/docs/src/guide/migration.md +++ b/docs/src/guide/migration.md @@ -8,29 +8,18 @@ migrate. ## 9.0.0 -* Newly created FTS / inverted indexes now default to format v4 instead of v1. - Format selection follows this priority: an explicit index creation parameter - `format_version`, then the `LANCE_FTS_FORMAT_VERSION` environment variable, - then v4 when neither is set. This preserves the environment variable as a - compatibility fallback for deployments that use it as a global rollout - switch. An invalid environment variable value fails index creation instead of - silently selecting v4; an explicit `format_version` takes precedence even - when the environment variable is invalid. - -* This affects users who create FTS / inverted indexes and need those indexes to - be readable by older Lance versions, or who depend on the v1 index layout. In - those cases, pass `format_version=1` when creating the index, or set - `LANCE_FTS_FORMAT_VERSION=1` for a process-wide rollout. When neither setting - is present, newly created indexes use v4, and older Lance readers may not be - able to read them. - - ```python - dataset.create_scalar_index("text", "INVERTED", format_version=1) - ``` - -* Existing v1 FTS indexes remain queryable. Operations that maintain an existing - v1 index, including append, incremental indexing, optimize, and mem-wal - maintained-index flush, should continue preserving the v1 format. +* Unless overridden, newly created FTS indexes use format v2. The code analyzer + and `block_size=256` require format v3, so readers must support v3 before an + index using either option is created. + +* To keep new indexes readable by nodes that support at most format v1 or v2, + set `format_version` in the index creation parameters, or set + `LANCE_FTS_FORMAT_VERSION` for a rollout-wide override. Formats v1 and v2 + require the text analyzer and `block_size=128`. + +* Operations that maintain an existing FTS index, including append, incremental + indexing, optimize, and mem-wal maintained-index flush, preserve its format + version. ## 7.2.0 diff --git a/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java b/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java index c03ce599270..ac23fd0d900 100755 --- a/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java +++ b/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java @@ -67,6 +67,7 @@ public static final class Builder { *
  • {@code "whitespace"}: splits tokens on whitespace *
  • {@code "raw"}: no tokenization *
  • {@code "ngram"}: N-Gram tokenizer + *
  • {@code "code"}: code-aware tokenizer *
  • {@code "icu"}: ICU dictionary-based Unicode word segmentation *
  • {@code "icu/split"}: ICU segmentation with simple-style delimiter splitting *
  • {@code "lindera/*"}: Lindera tokenizer @@ -233,8 +234,8 @@ public Builder prefixOnly(boolean prefixOnly) { *

    Supported values are {@code 128} and {@code 256}. New indexes default to {@code 128} when * this is not set. * - *

    {@code blockSize = 256} is experimental and may introduce breaking changes. Use {@code - * 128} when stable compatibility with the legacy posting layout is required. + *

    {@code blockSize = 256} requires FTS format v3. Format v3 also supports the default {@code + * blockSize = 128}. * * @param blockSize posting block size * @return this builder @@ -264,16 +265,18 @@ public Builder skipMerge(boolean skipMerge) { /** * Configure the on-disk FTS format version to write when creating a new index. * - *

    If unset, Lance uses {@code LANCE_FTS_FORMAT_VERSION} when present and otherwise writes - * v4. {@code formatVersion = 3} is experimental and is only valid with {@code blockSize = 256}. + *

    If unset, Lance uses {@code LANCE_FTS_FORMAT_VERSION} when present and otherwise selects + * v3 for the code analyzer or {@code blockSize = 256}, and v2 for other indexes. Format v3 + * supports both posting block sizes. Formats v1 and v2 support only {@code blockSize = 128} and + * cannot be used with the code analyzer. * - * @param formatVersion FTS format version, must be 1, 2, 3, or 4 + * @param formatVersion FTS format version, must be 1, 2, or 3 * @return this builder * @throws IllegalArgumentException */ public Builder formatVersion(int formatVersion) { - if (formatVersion != 1 && formatVersion != 2 && formatVersion != 3 && formatVersion != 4) { - throw new IllegalArgumentException("formatVersion must be 1, 2, 3, or 4"); + if (formatVersion != 1 && formatVersion != 2 && formatVersion != 3) { + throw new IllegalArgumentException("formatVersion must be 1, 2, or 3"); } this.formatVersion = formatVersion; return this; @@ -283,10 +286,10 @@ public Builder formatVersion(int formatVersion) { public ScalarIndexParams build() { if (formatVersion != null) { Preconditions.checkArgument( - formatVersion == 4 - || (blockSize == 256 && formatVersion == 3) - || (blockSize == 128 && formatVersion != 3), - "formatVersion 3 requires blockSize 256, and legacy formats require blockSize 128"); + formatVersion == 3 || blockSize == 128, "formatVersion 1 and 2 require blockSize 128"); + Preconditions.checkArgument( + !"code".equals(baseTokenizer) || formatVersion == 3, + "baseTokenizer 'code' requires formatVersion 3"); } Map params = new HashMap<>(); if (baseTokenizer != null) { diff --git a/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java b/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java index 8618d3ce091..2cbacb55bfe 100644 --- a/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java +++ b/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java @@ -60,30 +60,35 @@ void invalidBlockSizeIsRejected() { } @Test - void formatVersionThreeRequiresBlockSize256() { - ScalarIndexParams params = - InvertedIndexParams.builder().blockSize(256).formatVersion(3).build(); - - Map json = JsonUtils.fromJson(params.getJsonParams().orElseThrow()); - assertEquals(256, ((Number) json.get("block_size")).intValue()); - assertEquals(3, ((Number) json.get("format_version")).intValue()); + void formatVersionThreeSupportsBothBlockSizes() { + for (int blockSize : new int[] {128, 256}) { + ScalarIndexParams params = + InvertedIndexParams.builder().blockSize(blockSize).formatVersion(3).build(); + Map json = JsonUtils.fromJson(params.getJsonParams().orElseThrow()); + assertEquals(blockSize, ((Number) json.get("block_size")).intValue()); + assertEquals(3, ((Number) json.get("format_version")).intValue()); + } - assertThrows( - IllegalArgumentException.class, - () -> InvertedIndexParams.builder().formatVersion(3).build()); assertThrows( IllegalArgumentException.class, () -> InvertedIndexParams.builder().blockSize(256).formatVersion(2).build()); } @Test - void formatVersionFourSupportsBothBlockSizes() { - for (int blockSize : new int[] {128, 256}) { - ScalarIndexParams params = - InvertedIndexParams.builder().blockSize(blockSize).formatVersion(4).build(); - Map json = JsonUtils.fromJson(params.getJsonParams().orElseThrow()); - assertEquals(blockSize, ((Number) json.get("block_size")).intValue()); - assertEquals(4, ((Number) json.get("format_version")).intValue()); - } + void formatVersionFourIsRejected() { + assertThrows( + IllegalArgumentException.class, () -> InvertedIndexParams.builder().formatVersion(4)); + } + + @Test + void codeAnalyzerRequiresFormatVersionThreeWhenExplicit() { + ScalarIndexParams params = + InvertedIndexParams.builder().baseTokenizer("code").formatVersion(3).build(); + Map json = JsonUtils.fromJson(params.getJsonParams().orElseThrow()); + assertEquals(3, ((Number) json.get("format_version")).intValue()); + + assertThrows( + IllegalArgumentException.class, + () -> InvertedIndexParams.builder().baseTokenizer("code").formatVersion(2).build()); } } diff --git a/python/python/lance/dataset.py b/python/python/lance/dataset.py index fea973d1ee4..1e1927cf925 100644 --- a/python/python/lance/dataset.py +++ b/python/python/lance/dataset.py @@ -3370,11 +3370,10 @@ def create_scalar_index( format_version: int or str, optional This is for the ``INVERTED`` / ``FTS`` index. Explicit on-disk FTS format version to write when creating a new index. Accepts ``1``, - ``2``, ``3``, ``4``, ``"v1"``, ``"v2"``, ``"v3"``, or ``"v4"``. + ``2``, ``3``, ``"v1"``, ``"v2"``, or ``"v3"``. If unset, Lance uses ``LANCE_FTS_FORMAT_VERSION`` when present and - otherwise writes v4. - ``format_version=3`` is experimental and is only valid with - ``block_size=256``. + otherwise writes v2 for text analysis with ``block_size=128`` and + v3 for code analysis or ``block_size=256``. with_position: bool, default False This is for the ``INVERTED`` index. If True, the index will store the diff --git a/python/python/tests/test_scalar_index.py b/python/python/tests/test_scalar_index.py index c50014246c0..552fbe1a1ad 100644 --- a/python/python/tests/test_scalar_index.py +++ b/python/python/tests/test_scalar_index.py @@ -809,10 +809,17 @@ def test_full_text_search(dataset, with_position, base_tokenizer): ) -def test_code_analyzer_does_not_split_identifiers_by_default(tmp_path): +@pytest.mark.parametrize("block_size", [128, 256]) +def test_code_analyzer_does_not_split_identifiers_by_default(tmp_path, block_size): table = pa.table({"code": ["GetUserName", "GetUserEmail", "user"]}) ds = lance.write_dataset(table, tmp_path) - ds.create_scalar_index("code", index_type="INVERTED", analyzer="code") + ds.create_scalar_index( + "code", + index_type="INVERTED", + analyzer="code", + block_size=block_size, + ) + assert ds.describe_indices()[0].segments[0].index_version == 3 results = ds.to_table( columns=["code"], @@ -927,6 +934,19 @@ def test_code_analyzer_flags_require_code_analyzer(tmp_path): ) +def test_code_analyzer_requires_fts_v3(tmp_path): + table = pa.table({"code": ["getUserName"]}) + ds = lance.write_dataset(table, tmp_path) + + with pytest.raises(ValueError, match="requires FTS format_version=3"): + ds.create_scalar_index( + "code", + index_type="INVERTED", + analyzer="code", + format_version=2, + ) + + def test_code_analyzer_complex_code_constructs(tmp_path): table = pa.table( { @@ -1273,7 +1293,7 @@ def test_create_scalar_index_fts_block_size(dataset): ) indices = dataset.describe_indices() doc_index = next(index for index in indices if index.name == "doc_idx") - assert doc_index.segments[0].index_version == 4 + assert doc_index.segments[0].index_version == 3 row = dataset.take(indices=[0], columns=["doc"]) query = row.column(0)[0].as_py().split(" ")[0] @@ -5392,7 +5412,7 @@ def test_json_inverted_match_query(tmp_path): @pytest.mark.parametrize( ("format_version", "expected_format_version"), - [(1, 1), (2, 2), (4, 4), ("v1", 1), ("v2", 2), ("v4", 4)], + [(1, 1), (2, 2), (3, 3), ("v1", 1), ("v2", 2), ("v3", 3)], ) def test_describe_indices(tmp_path, format_version, expected_format_version): data = pa.table( @@ -5568,8 +5588,8 @@ def _run_fts_format_creation_probe( [ ("1", {}, 1), ("2", {}, 2), + ("3", {}, 3), ("3", {"block_size": 256}, 3), - ("4", {}, 4), ], ) def test_create_inverted_index_uses_env_format_version( @@ -5596,8 +5616,8 @@ def test_create_inverted_index_explicit_format_version_overrides_env(tmp_path): assert result.returncode == 0, result.stderr -def test_create_inverted_index_defaults_to_v4_without_env(tmp_path): - result = _run_fts_format_creation_probe(tmp_path, None, expected_format_version=4) +def test_create_text_inverted_index_defaults_to_v2_without_env(tmp_path): + result = _run_fts_format_creation_probe(tmp_path, None, expected_format_version=2) assert result.returncode == 0, result.stderr @@ -5617,8 +5637,8 @@ def test_create_inverted_index_rejects_invalid_format_version(tmp_path): with pytest.raises(ValueError, match="unsupported FTS format version"): ds.create_scalar_index("text", index_type="INVERTED", format_version="v5") - with pytest.raises(ValueError, match="format_version=3"): - ds.create_scalar_index("text", index_type="INVERTED", format_version="v3") + with pytest.raises(ValueError, match="unsupported FTS format version"): + ds.create_scalar_index("text", index_type="INVERTED", format_version="v4") def test_vector_filter_fts_search(tmp_path): diff --git a/python/src/dataset.rs b/python/src/dataset.rs index eddf555fd35..07dbf327a05 100644 --- a/python/src/dataset.rs +++ b/python/src/dataset.rs @@ -2583,7 +2583,7 @@ impl Dataset { value.to_string() } else { return Err(PyValueError::new_err( - "format_version must be 1, 2, 3, 4, 'v1', 'v2', 'v3', or 'v4'", + "format_version must be 1, 2, 3, 'v1', 'v2', or 'v3'", )); }; let format_version = value diff --git a/rust/lance-index/src/scalar/inverted.rs b/rust/lance-index/src/scalar/inverted.rs index ca27367febc..72604243c1c 100644 --- a/rust/lance-index/src/scalar/inverted.rs +++ b/rust/lance-index/src/scalar/inverted.rs @@ -268,7 +268,7 @@ impl ScalarIndexPlugin for InvertedIndexPlugin { } fn version(&self) -> u32 { - INVERTED_INDEX_VERSION_V4 + INVERTED_INDEX_VERSION_V3 } fn new_query_parser( @@ -320,9 +320,9 @@ mod tests { use crate::scalar::{BuiltinIndexType, ScalarIndexParams}; #[test] - fn test_plugin_version_tracks_current_index_version() { + fn test_plugin_version_tracks_v3_capability_gate() { let plugin = InvertedIndexPlugin; - assert_eq!(plugin.version(), INVERTED_INDEX_VERSION_V4); + assert_eq!(plugin.version(), INVERTED_INDEX_VERSION_V3); } #[test] diff --git a/rust/lance-index/src/scalar/inverted/builder.rs b/rust/lance-index/src/scalar/inverted/builder.rs index 5cf754dfa3a..c90f96053ef 100644 --- a/rust/lance-index/src/scalar/inverted/builder.rs +++ b/rust/lance-index/src/scalar/inverted/builder.rs @@ -1820,16 +1820,16 @@ pub(crate) fn inverted_list_schema_for_version_with_block_size_and_impacts( InvertedListFormatVersion::V1 => { inverted_list_schema_v1(with_position, block_size, with_impacts) } - InvertedListFormatVersion::V2 - | InvertedListFormatVersion::V3 - | InvertedListFormatVersion::V4 => inverted_list_schema_with_tail_codec_and_position_codec( - with_position, - format_version, - PostingTailCodec::VarintDelta, - Some(PositionStreamCodec::PackedDelta), - block_size, - with_impacts, - ), + InvertedListFormatVersion::V2 | InvertedListFormatVersion::V3 => { + inverted_list_schema_with_tail_codec_and_position_codec( + with_position, + format_version, + PostingTailCodec::VarintDelta, + Some(PositionStreamCodec::PackedDelta), + block_size, + with_impacts, + ) + } } } @@ -2927,7 +2927,7 @@ mod tests { assert_eq!(written_partitions, remapped_partitions); assert_eq!( parse_format_version_from_metadata(metadata)?, - InvertedListFormatVersion::V4 + InvertedListFormatVersion::V2 ); for (new_id, old_id) in expected_partitions.iter().enumerate() { @@ -3452,6 +3452,39 @@ mod tests { assert_eq!(builder.posting_tail_codec, PostingTailCodec::VarintDelta); } + #[test] + fn test_v3_128_reuses_v2_physical_layout() { + for with_position in [false, true] { + for with_impacts in [false, true] { + let v2 = inverted_list_schema_for_version_with_block_size_and_impacts( + with_position, + InvertedListFormatVersion::V2, + LEGACY_BLOCK_SIZE, + with_impacts, + ); + let v3 = inverted_list_schema_for_version_with_block_size_and_impacts( + with_position, + InvertedListFormatVersion::V3, + LEGACY_BLOCK_SIZE, + with_impacts, + ); + + assert_eq!(v2.fields(), v3.fields()); + let mut v2_metadata = v2.metadata.clone(); + let mut v3_metadata = v3.metadata.clone(); + assert_eq!( + v2_metadata.remove(FTS_FORMAT_VERSION_KEY).as_deref(), + Some("2") + ); + assert_eq!( + v3_metadata.remove(FTS_FORMAT_VERSION_KEY).as_deref(), + Some("3") + ); + assert_eq!(v2_metadata, v3_metadata); + } + } + } + #[tokio::test] async fn test_inverted_index_without_positions_tracks_frequency() -> Result<()> { let index_dir = TempDir::default(); diff --git a/rust/lance-index/src/scalar/inverted/encoding.rs b/rust/lance-index/src/scalar/inverted/encoding.rs index 6efa6c0b7b3..6d57b032650 100644 --- a/rust/lance-index/src/scalar/inverted/encoding.rs +++ b/rust/lance-index/src/scalar/inverted/encoding.rs @@ -128,8 +128,8 @@ pub fn compress_posting_list_with_tail_codec_and_block_size<'a>( } /// Byte length of the block-max-score prefix on posting blocks. 128-doc -/// blocks store a per-block max score, patched in at build time; 256-doc -/// (V3) blocks always carry impact skip data, which supersedes it, so they +/// blocks store a per-block max score, patched in at build time; 256-document +/// blocks always carry impact skip data, which supersedes it, so they /// store none. #[inline] pub fn posting_block_score_prefix_len(block_size: usize) -> usize { @@ -167,7 +167,7 @@ fn encode_posting_block_payload( compress_sorted_block_with::(doc_ids, &mut buffer, block)?; compress_block_with::(frequencies, &mut buffer, block)?; } - // 256-doc blocks (format V3) store frequencies with patched FOR: + // 256-document blocks store frequencies with patched FOR: // outliers no longer widen the whole block, which matters because one // large tf per block otherwise doubles the frequency payload. BitPacker8x::BLOCK_LEN => { diff --git a/rust/lance-index/src/scalar/inverted/impact.rs b/rust/lance-index/src/scalar/inverted/impact.rs index 505d9f613d9..848c2376459 100644 --- a/rust/lance-index/src/scalar/inverted/impact.rs +++ b/rust/lance-index/src/scalar/inverted/impact.rs @@ -994,7 +994,7 @@ mod tests { } #[test] - fn v2_and_v3_impacts_use_identical_encoding() { + fn posting_block_sizes_use_identical_impact_encoding() { let docs = vec![(3, 1, 100), (9, 2, 40), (200, 7, 80)]; let mut v2_builder = ImpactSkipDataBuilder::with_capacity(1, 128); v2_builder.append_block(&docs).unwrap(); diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs index e04d6d1924a..a6c1aad5a9b 100644 --- a/rust/lance-index/src/scalar/inverted/index.rs +++ b/rust/lance-index/src/scalar/inverted/index.rs @@ -89,12 +89,10 @@ use std::str::FromStr; // Version 0: Arrow TokenSetFormat (legacy) // Version 1: Fst TokenSetFormat with per-doc compressed positions // Version 2: Fst TokenSetFormat with shared posting-list position streams. -// Version 3: Version 2 layout with 256-document physical posting blocks. -// Version 4: Code analyzer support with configurable posting block sizes. +// Version 3: Version 2 layout with configurable posting blocks and analyzer metadata. pub const INVERTED_INDEX_VERSION_V1: u32 = 1; pub const INVERTED_INDEX_VERSION_V2: u32 = 2; pub const INVERTED_INDEX_VERSION_V3: u32 = 3; -pub const INVERTED_INDEX_VERSION_V4: u32 = 4; pub const TOKENS_FILE: &str = "tokens.lance"; pub const INVERT_LIST_FILE: &str = "invert.lance"; pub const DOCS_FILE: &str = "docs.lance"; @@ -149,7 +147,7 @@ pub fn resolve_fts_format_version( } pub fn default_fts_format_version() -> InvertedListFormatVersion { - InvertedListFormatVersion::V4 + InvertedListFormatVersion::V2 } pub fn current_fts_format_version() -> InvertedListFormatVersion { @@ -157,16 +155,15 @@ pub fn current_fts_format_version() -> InvertedListFormatVersion { } pub fn max_supported_fts_format_version() -> InvertedListFormatVersion { - InvertedListFormatVersion::V4 + InvertedListFormatVersion::V3 } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] pub enum InvertedListFormatVersion { V1, + #[default] V2, V3, - #[default] - V4, } impl InvertedListFormatVersion { @@ -202,26 +199,25 @@ impl InvertedListFormatVersion { Self::V1 => INVERTED_INDEX_VERSION_V1, Self::V2 => INVERTED_INDEX_VERSION_V2, Self::V3 => INVERTED_INDEX_VERSION_V3, - Self::V4 => INVERTED_INDEX_VERSION_V4, } } pub fn posting_tail_codec(self) -> PostingTailCodec { match self { Self::V1 => PostingTailCodec::Fixed32, - Self::V2 | Self::V3 | Self::V4 => PostingTailCodec::VarintDelta, + Self::V2 | Self::V3 => PostingTailCodec::VarintDelta, } } pub fn position_codec(self) -> Option { match self { Self::V1 => None, - Self::V2 | Self::V3 | Self::V4 => Some(PositionStreamCodec::PackedDelta), + Self::V2 | Self::V3 => Some(PositionStreamCodec::PackedDelta), } } pub fn uses_shared_position_stream(self) -> bool { - matches!(self, Self::V2 | Self::V3 | Self::V4) + matches!(self, Self::V2 | Self::V3) } } @@ -233,9 +229,8 @@ impl FromStr for InvertedListFormatVersion { "1" | "v1" | "V1" => Ok(Self::V1), "2" | "v2" | "V2" => Ok(Self::V2), "3" | "v3" | "V3" => Ok(Self::V3), - "4" | "v4" | "V4" => Ok(Self::V4), other => Err(Error::index(format!( - "unsupported FTS format version {}, expected 1, 2, 3, or 4", + "unsupported FTS format version {}, expected 1, 2, or 3", other ))), } @@ -246,7 +241,11 @@ pub fn default_fts_format_version_for_block_size( block_size: usize, ) -> Result { validate_block_size(block_size)?; - Ok(InvertedListFormatVersion::V4) + match block_size { + LEGACY_BLOCK_SIZE => Ok(InvertedListFormatVersion::V2), + 256 => Ok(InvertedListFormatVersion::V3), + _ => unreachable!("validate_block_size limits supported block sizes"), + } } pub fn validate_format_version_block_size( @@ -256,17 +255,13 @@ pub fn validate_format_version_block_size( validate_block_size(block_size)?; match (format_version, block_size) { (InvertedListFormatVersion::V1 | InvertedListFormatVersion::V2, LEGACY_BLOCK_SIZE) - | (InvertedListFormatVersion::V3, 256) - | (InvertedListFormatVersion::V4, _) => Ok(()), + | (InvertedListFormatVersion::V3, _) => Ok(()), (InvertedListFormatVersion::V1 | InvertedListFormatVersion::V2, 256) => { Err(Error::invalid_input(format!( "FTS format_version={} is incompatible with block_size=256; use format_version=3", format_version.index_version() ))) } - (InvertedListFormatVersion::V3, other) => Err(Error::invalid_input(format!( - "FTS format_version=3 requires block_size=256, got {other}" - ))), _ => unreachable!("validate_block_size limits supported block sizes"), } } @@ -449,7 +444,8 @@ pub(super) fn parse_format_version_from_metadata( ) -> Result { if let Some(value) = metadata.get(FTS_FORMAT_VERSION_KEY) { let format_version = InvertedListFormatVersion::from_str(value)?; - validate_format_version_block_size(format_version, parse_posting_block_size(metadata)?)?; + let block_size = parse_posting_block_size(metadata)?; + validate_format_version_block_size(format_version, block_size)?; return Ok(format_version); } let block_size = parse_posting_block_size(metadata)?; @@ -1729,7 +1725,7 @@ impl InvertedPartition { num_docs, false, frag_reuse_index, - // V3 (256-doc block) partitions score with quantized doc lengths. + // 256-document blocks score with quantized document lengths. inverted_list.block_size() == MAX_POSTING_BLOCK_SIZE, )); @@ -4942,7 +4938,7 @@ impl CompressedPostingList { } pub fn block_max_score(&self, block_idx: usize) -> f32 { - // 256-doc (V3) blocks store no per-block max score: their impact + // 256-document blocks store no per-block max score: their impact // skip data supplies the tight per-block bound, so callers on that // path never reach here. Fall back to the list-level max, which is // still a valid (looser) bound for any block. @@ -6262,11 +6258,11 @@ impl Ord for RawDocInfo { } } -/// Lucene SmallFloat-style doc-length quantization for V3 scoring and impact +/// Lucene SmallFloat-style document-length quantization for 256-document-block scoring and impact /// norms: a 4-mantissa-bit float-like byte code. Values 0-7 are exact; larger /// values keep their top four significand bits (relative error <= 6.25%) and /// decode to their bucket floor. The floor only ever shortens a doc, so impact -/// bounds remain conservative for exact-scoring V2 as well as quantized V3. +/// bounds remain conservative for exact scoring as well as quantized scoring. pub(super) fn quantize_doc_length(value: u32) -> u8 { let num_bits = 32 - value.leading_zeros(); if num_bits < 4 { @@ -6381,7 +6377,7 @@ pub struct DocSet { total_tokens: u64, - // V3 (256-doc block) partitions score with quantized doc lengths: the + // 256-document-block partitions score with quantized document lengths: the // flag is set at partition load and the byte-norm slab bakes lazily on // first scoring use (shared by clones of the loaded set). 128-block // partitions never set the flag and keep exact scoring. @@ -6697,13 +6693,13 @@ impl DocSet { self.num_tokens[doc_id as usize] } - /// Enable quantized doc-length scoring (V3 / 256-doc block partitions). + /// Enable quantized document-length scoring for 256-document-block partitions. pub fn set_quantized_scoring(&mut self, quantized: bool) { self.scoring_quantized = quantized; } - /// The quantized doc-length slab when this set scores quantized (V3 - /// partitions), baked on first use; `None` for exact-scoring sets. + /// The quantized document-length slab when this set scores quantized, + /// baked on first use; `None` for exact-scoring sets. pub fn scoring_norms(&self) -> Option<&[u8]> { if !self.scoring_quantized { return None; @@ -6720,8 +6716,8 @@ impl DocSet { ) } - /// Doc length as scoring sees it: the quantized bucket floor for V3 - /// partitions, the exact value otherwise. + /// Document length as scoring sees it: the quantized bucket floor for + /// 256-document-block partitions, the exact value otherwise. #[inline] pub fn scoring_num_tokens(&self, doc_id: u32) -> u32 { match self.scoring_norms() { @@ -8034,10 +8030,10 @@ mod tests { } #[test] - fn test_resolve_fts_format_version_defaults_to_v4() { + fn test_resolve_fts_format_version_defaults_to_v2() { assert_eq!( resolve_fts_format_version(None).unwrap(), - InvertedListFormatVersion::V4 + InvertedListFormatVersion::V2 ); assert_eq!( resolve_fts_format_version(Some("2")).unwrap(), @@ -8047,10 +8043,7 @@ mod tests { resolve_fts_format_version(Some("3")).unwrap(), InvertedListFormatVersion::V3 ); - assert_eq!( - resolve_fts_format_version(Some("4")).unwrap(), - InvertedListFormatVersion::V4 - ); + assert!(resolve_fts_format_version(Some("4")).is_err()); } #[test] @@ -8753,8 +8746,6 @@ mod tests { #[case::v1(InvertedListFormatVersion::V1, LEGACY_BLOCK_SIZE)] #[case::v2(InvertedListFormatVersion::V2, LEGACY_BLOCK_SIZE)] #[case::v3(InvertedListFormatVersion::V3, 256)] - #[case::v4_128(InvertedListFormatVersion::V4, LEGACY_BLOCK_SIZE)] - #[case::v4_256(InvertedListFormatVersion::V4, 256)] #[tokio::test] async fn test_prewarm_streams_in_chunks_preserves_content( #[case] format_version: InvertedListFormatVersion, @@ -11435,7 +11426,7 @@ mod tests { } #[tokio::test] - async fn test_block_size_256_writes_v4_metadata_and_index_version() -> Result<()> { + async fn test_block_size_256_writes_v3_metadata_and_index_version() -> Result<()> { let src_dir = TempObjDir::default(); let dest_dir = TempObjDir::default(); let src_store = Arc::new(LanceIndexStore::new( @@ -11451,7 +11442,7 @@ mod tests { let params = InvertedIndexParams::default().block_size(256)?; let format_version = params.resolved_format_version(); - assert_eq!(format_version, InvertedListFormatVersion::V4); + assert_eq!(format_version, InvertedListFormatVersion::V3); let mut partition = InnerBuilder::new_with_format_version_and_block_size( 0, @@ -11474,17 +11465,17 @@ mod tests { write_test_metadata(&src_store, vec![0], params).await; let index = InvertedIndex::load(src_store, None, &LanceCache::no_cache()).await?; - assert_eq!(index.format_version(), InvertedListFormatVersion::V4); - assert_eq!(index.index_version(), INVERTED_INDEX_VERSION_V4); + assert_eq!(index.format_version(), InvertedListFormatVersion::V3); + assert_eq!(index.index_version(), INVERTED_INDEX_VERSION_V3); let created = index .update(empty_doc_stream(), dest_store.as_ref(), None) .await?; - assert_eq!(created.index_version, INVERTED_INDEX_VERSION_V4); + assert_eq!(created.index_version, INVERTED_INDEX_VERSION_V3); let updated = InvertedIndex::load(dest_store, None, &LanceCache::no_cache()).await?; - assert_eq!(updated.format_version(), InvertedListFormatVersion::V4); - assert_eq!(updated.index_version(), INVERTED_INDEX_VERSION_V4); + assert_eq!(updated.format_version(), InvertedListFormatVersion::V3); + assert_eq!(updated.index_version(), INVERTED_INDEX_VERSION_V3); Ok(()) } @@ -11542,9 +11533,8 @@ mod tests { #[rstest::rstest] #[case::v1(InvertedListFormatVersion::V1, LEGACY_BLOCK_SIZE)] #[case::v2(InvertedListFormatVersion::V2, LEGACY_BLOCK_SIZE)] - #[case::v3(InvertedListFormatVersion::V3, 256)] - #[case::v4_128(InvertedListFormatVersion::V4, LEGACY_BLOCK_SIZE)] - #[case::v4_256(InvertedListFormatVersion::V4, 256)] + #[case::v3_128(InvertedListFormatVersion::V3, LEGACY_BLOCK_SIZE)] + #[case::v3_256(InvertedListFormatVersion::V3, 256)] #[tokio::test] async fn test_merge_segments_preserves_format_version( #[case] format_version: InvertedListFormatVersion, diff --git a/rust/lance-index/src/scalar/inverted/lazy_docset.rs b/rust/lance-index/src/scalar/inverted/lazy_docset.rs index d062cca4f63..025c91cde21 100644 --- a/rust/lance-index/src/scalar/inverted/lazy_docset.rs +++ b/rust/lance-index/src/scalar/inverted/lazy_docset.rs @@ -75,7 +75,7 @@ pub struct DeferredDocSet { docs_path: String, is_legacy: bool, frag_reuse_index: Option>, - /// V3 (256-doc block) partitions score with quantized doc lengths; the + /// 256-document-block partitions score with quantized document lengths; the /// flag is applied to every `DocSet` this deferred set materializes. quantized_scoring: bool, /// Doc count cached at construction so `len()` stays sync + IO-free. diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs index cc502609c94..a55c5b9d16f 100644 --- a/rust/lance-index/src/scalar/inverted/tokenizer.rs +++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs @@ -160,10 +160,9 @@ pub struct InvertedIndexParams { /// /// This is a build-time only parameter and is not persisted with the index. /// If unset, new index creation falls back to - /// `LANCE_FTS_FORMAT_VERSION`, then v4 when the environment variable is - /// also unset. - /// `format_version = 3` is experimental and is only valid with - /// `block_size = 256`. + /// `LANCE_FTS_FORMAT_VERSION`. Without either override, text analysis with + /// 128-document blocks writes v2, while code analysis or 256-document blocks + /// write v3. #[serde( rename = "format_version", skip_serializing, @@ -486,7 +485,7 @@ where serde_json::Value::Number(value) => { let Some(format_version) = value.as_u64() else { return Err(serde::de::Error::custom(format!( - "FTS format_version must be 1, 2, 3, or 4, got {value}" + "FTS format_version must be 1, 2, or 3, got {value}" ))); }; resolve_fts_format_version(Some(&format_version.to_string())) @@ -494,13 +493,14 @@ where .map_err(serde::de::Error::custom) } other => Err(serde::de::Error::custom(format!( - "FTS format_version must be 1, 2, 3, or 4, got {other}" + "FTS format_version must be 1, 2, or 3, got {other}" ))), } } fn resolve_creation_format_version( explicit: Option, + default: InvertedListFormatVersion, ) -> Result { if let Some(format_version) = explicit { return Ok(format_version); @@ -512,9 +512,9 @@ fn resolve_creation_format_version( "invalid {LANCE_FTS_FORMAT_VERSION_ENV_KEY} value {value:?}: {err}" )) }), - Err(env::VarError::NotPresent) => resolve_fts_format_version(None), + Err(env::VarError::NotPresent) => Ok(default), Err(env::VarError::NotUnicode(value)) => Err(Error::invalid_input(format!( - "invalid {LANCE_FTS_FORMAT_VERSION_ENV_KEY} value {value:?}: expected UTF-8 value 1, 2, 3, or 4" + "invalid {LANCE_FTS_FORMAT_VERSION_ENV_KEY} value {value:?}: expected UTF-8 value 1, 2, or 3" ))), } } @@ -820,29 +820,40 @@ impl InvertedIndexParams { /// Set the on-disk FTS format version to use when creating a new index. /// /// If unset, new index creation falls back to - /// `LANCE_FTS_FORMAT_VERSION`, then v4 when the environment variable is - /// also unset. Existing indexes keep their own on-disk format during - /// update and optimize operations. - /// `format_version = 3` is experimental and is only valid with - /// `block_size = 256`. + /// `LANCE_FTS_FORMAT_VERSION`. Without either override, text analysis with + /// 128-document blocks writes v2, while code analysis or 256-document blocks + /// write v3. Existing indexes keep their own format during update and + /// optimize operations. pub fn format_version(mut self, format_version: InvertedListFormatVersion) -> Self { self.format_version = Some(format_version); self } /// Resolve the requested FTS format version, falling back to the default for - /// the configured block size. + /// the configured analyzer and block size. pub fn resolved_format_version(&self) -> InvertedListFormatVersion { self.format_version.unwrap_or_else(|| { - default_fts_format_version_for_block_size(self.block_size) - .expect("InvertedIndexParams block_size must be validated before use") + if self.base_tokenizer == "code" { + InvertedListFormatVersion::V3 + } else { + default_fts_format_version_for_block_size(self.block_size) + .expect("InvertedIndexParams block_size must be validated before use") + } }) } /// Validate that the requested FTS format version can safely encode the /// configured posting block size. pub fn validate_format_version(&self) -> Result<()> { - validate_format_version_block_size(self.resolved_format_version(), self.block_size) + let format_version = self.resolved_format_version(); + validate_format_version_block_size(format_version, self.block_size)?; + if self.base_tokenizer == "code" && format_version != InvertedListFormatVersion::V3 { + return Err(Error::invalid_input(format!( + "base_tokenizer='code' requires FTS format_version=3, got {}", + format_version.index_version() + ))); + } + Ok(()) } /// Serialize params for the build/training path, including build-only fields. @@ -873,7 +884,7 @@ impl InvertedIndexParams { } /// Deserialize params for new index training, using the environment - /// compatibility fallback and current creation defaults for omitted fields. + /// override and current creation defaults for omitted fields. pub(crate) fn from_training_json(params: &str) -> Result { let supplied = serde_json::from_str::(params)?; let mut value = serde_json::to_value(Self::default())?; @@ -887,7 +898,11 @@ impl InvertedIndexParams { object.extend(supplied.clone()); let mut params: Self = serde_json::from_value(value)?; - params.format_version = Some(resolve_creation_format_version(params.format_version)?); + let default_format_version = params.resolved_format_version(); + params.format_version = Some(resolve_creation_format_version( + params.format_version, + default_format_version, + )?); params.validate_format_version()?; Ok(params) } @@ -1100,10 +1115,10 @@ mod tests { } #[test] - fn test_default_format_version_resolves_to_v4() { + fn test_default_format_version_resolves_to_v2() { assert_eq!( InvertedIndexParams::default().resolved_format_version(), - InvertedListFormatVersion::V4 + InvertedListFormatVersion::V2 ); } @@ -1154,16 +1169,46 @@ mod tests { } #[test] - fn test_block_size_256_defaults_to_v4() { + fn test_block_size_256_defaults_to_v3() { assert_eq!( InvertedIndexParams::default() .block_size(256) .unwrap() .resolved_format_version(), - InvertedListFormatVersion::V4 + InvertedListFormatVersion::V3 + ); + } + + #[test] + fn test_code_analyzer_defaults_to_v3_for_supported_block_sizes() { + assert_eq!( + InvertedIndexParams::code().resolved_format_version(), + InvertedListFormatVersion::V3 + ); + assert_eq!( + InvertedIndexParams::code() + .block_size(256) + .unwrap() + .resolved_format_version(), + InvertedListFormatVersion::V3 ); } + #[test] + fn test_training_json_uses_v3_for_code_analyzer_and_supported_block_sizes() { + for block_size in [128, 256] { + let params = InvertedIndexParams::from_training_json( + &serde_json::json!({ + "base_tokenizer": "code", + "block_size": block_size, + }) + .to_string(), + ) + .unwrap(); + assert_eq!(params.format_version, Some(InvertedListFormatVersion::V3)); + } + } + #[test] fn test_text_analyzer_replaces_code_defaults() { let params = InvertedIndexParams::code().analyzer("text").unwrap(); @@ -1305,13 +1350,13 @@ mod tests { .validate_format_version() .unwrap(); InvertedIndexParams::default() - .format_version(InvertedListFormatVersion::V4) + .format_version(InvertedListFormatVersion::V3) .validate_format_version() .unwrap(); InvertedIndexParams::default() .block_size(256) .unwrap() - .format_version(InvertedListFormatVersion::V4) + .format_version(InvertedListFormatVersion::V3) .validate_format_version() .unwrap(); @@ -1323,11 +1368,11 @@ mod tests { .unwrap_err(); assert!(err.to_string().contains("block_size=256")); - let err = InvertedIndexParams::default() - .format_version(InvertedListFormatVersion::V3) + let err = InvertedIndexParams::code() + .format_version(InvertedListFormatVersion::V2) .validate_format_version() .unwrap_err(); - assert!(err.to_string().contains("format_version=3")); + assert!(err.to_string().contains("requires FTS format_version=3")); } #[test] diff --git a/rust/lance-index/src/scalar/inverted/wand.rs b/rust/lance-index/src/scalar/inverted/wand.rs index e2eb352175b..b6e2972bdbe 100644 --- a/rust/lance-index/src/scalar/inverted/wand.rs +++ b/rust/lance-index/src/scalar/inverted/wand.rs @@ -555,7 +555,7 @@ impl BlockMaxWindow { }; } - // V3 postings score quantized doc lengths, which can be shorter than + // 256-document-block postings score quantized document lengths, which can be shorter than // the exact lengths used to bake a legacy max score. Without impacts, // use the score-independent BM25 ceiling instead of that stale bound. if list.block_size == MAX_POSTING_BLOCK_SIZE { @@ -1752,7 +1752,7 @@ impl<'a, S: Scorer> Wand<'a, S> { /// Per-search norm→BM25-denominator cache (Lucene's norm cache): the doc /// byte-norm slab plus the 256 possible denominator addends. Available - /// when the DocSet scores quantized (V3 partitions) and the scorer + /// when the DocSet scores quantized (256-document-block partitions) and the scorer /// factors `doc_weight` as `(K1+1)*freq/(freq + addend)`. Scoring through /// the cache is bit-identical to `scorer.doc_weight`, because both /// evaluate the same expressions on the same quantized lengths. @@ -6079,7 +6079,7 @@ mod tests { } #[test] - fn test_v3_without_impacts_uses_conservative_quantized_score_bound() { + fn test_256_document_blocks_without_impacts_use_conservative_quantized_score_bound() { let exact_doc_length = 300; let quantized_doc_length = super::super::index::dequantize_doc_length( super::super::index::quantize_doc_length(exact_doc_length), @@ -6125,7 +6125,7 @@ mod tests { } #[test] - fn test_v3_without_impacts_unknown_scorer_uses_infinite_bound() { + fn test_256_document_blocks_without_impacts_unknown_scorer_uses_infinite_bound() { let doc_ids = [0_u32]; let frequencies = [10_u32]; let blocks = compress_posting_list_with_tail_codec_and_block_size( diff --git a/rust/lance/src/dataset/mem_wal/index.rs b/rust/lance/src/dataset/mem_wal/index.rs index e67e73d92a6..072d8e7d5a3 100644 --- a/rust/lance/src/dataset/mem_wal/index.rs +++ b/rust/lance/src/dataset/mem_wal/index.rs @@ -209,9 +209,8 @@ impl MemIndexConfig { 0 | 1 => Ok(InvertedListFormatVersion::V1), 2 => Ok(InvertedListFormatVersion::V2), 3 => Ok(InvertedListFormatVersion::V3), - 4 => Ok(InvertedListFormatVersion::V4), version => Err(Error::invalid_input(format!( - "FTS index '{}' has unsupported index_version {}; expected 0, 1, 2, 3, or 4", + "FTS index '{}' has unsupported index_version {}; expected 0, 1, 2, or 3", index_meta.name, version ))), } @@ -1385,7 +1384,7 @@ mod tests { (0, InvertedListFormatVersion::V1), (1, InvertedListFormatVersion::V1), (2, InvertedListFormatVersion::V2), - (4, InvertedListFormatVersion::V4), + (3, InvertedListFormatVersion::V3), ] { let config = MemIndexConfig::fts_from_metadata(&fts_index_metadata(index_version), &schema) @@ -1408,44 +1407,32 @@ mod tests { let arrow_schema = create_test_schema(); let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - let err = MemIndexConfig::fts_from_metadata(&fts_index_metadata(5), &schema).unwrap_err(); + let err = MemIndexConfig::fts_from_metadata(&fts_index_metadata(4), &schema).unwrap_err(); assert!( - err.to_string().contains("unsupported index_version 5"), + err.to_string().contains("unsupported index_version 4"), "{err}" ); } #[test] - fn fts_from_metadata_rejects_v3_without_256_block_size() { + fn fts_from_metadata_accepts_v3_with_legacy_block_size() { let arrow_schema = create_test_schema(); let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - let missing_details = - MemIndexConfig::fts_from_metadata(&fts_index_metadata_with_details(3, None), &schema) - .unwrap_err(); - assert!( - missing_details - .to_string() - .contains("requires block_size=256"), - "{missing_details}" - ); - assert!( - missing_details.to_string().contains("got 128"), - "{missing_details}" - ); - - let default_details = - MemIndexConfig::fts_from_metadata(&fts_index_metadata(3), &schema).unwrap_err(); - assert!( - default_details - .to_string() - .contains("requires block_size=256"), - "{default_details}" - ); - assert!( - default_details.to_string().contains("got 128"), - "{default_details}" - ); + for metadata in [ + fts_index_metadata_with_details(3, None), + fts_index_metadata(3), + ] { + let config = MemIndexConfig::fts_from_metadata(&metadata, &schema).unwrap(); + let MemIndexConfig::Fts(config) = config else { + unreachable!("FTS metadata should create an FTS config"); + }; + assert_eq!( + config.params.resolved_format_version(), + InvertedListFormatVersion::V3 + ); + assert_eq!(config.params.posting_block_size(), 128); + } } #[test] diff --git a/rust/lance/src/dataset/mem_wal/memtable/flush.rs b/rust/lance/src/dataset/mem_wal/memtable/flush.rs index 72867700dde..a3607ad61ac 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/flush.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/flush.rs @@ -1192,7 +1192,7 @@ mod tests { use super::*; use arrow_array::{Int32Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; - use lance_index::scalar::inverted::INVERTED_INDEX_VERSION_V4; + use lance_index::scalar::inverted::INVERTED_INDEX_VERSION_V2; use std::sync::Arc; use tempfile::TempDir; @@ -2193,7 +2193,7 @@ mod tests { assert_eq!(indices.len(), 1); assert_eq!(indices[0].name, "text_fts"); - assert_eq!(indices[0].index_version, INVERTED_INDEX_VERSION_V4 as i32); + assert_eq!(indices[0].index_version, INVERTED_INDEX_VERSION_V2 as i32); // Verify FTS query returns correct results // Searching for "hello" should find the first document diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs index 513bb68b7b6..a0eb824c5b0 100644 --- a/rust/lance/src/index.rs +++ b/rust/lance/src/index.rs @@ -2892,9 +2892,9 @@ mod tests { use lance_core::utils::tempfile::TempStrDir; use lance_datagen::gen_batch; use lance_datagen::{BatchCount, ByteCount, Dimension, RowCount, array}; - use lance_index::pbold::BTreeIndexDetails; + use lance_index::pbold::{BTreeIndexDetails, InvertedIndexDetails}; use lance_index::scalar::bitmap::BITMAP_LOOKUP_NAME; - use lance_index::scalar::inverted::INVERTED_INDEX_VERSION_V4; + use lance_index::scalar::inverted::INVERTED_INDEX_VERSION_V3; use lance_index::scalar::{ BuiltinIndexType, FullTextSearchQuery, InvertedIndexParams, ScalarIndexParams, }; @@ -5004,8 +5004,11 @@ mod tests { ); } + #[rstest] + #[case::block_size_128(128)] + #[case::block_size_256(256)] #[tokio::test] - async fn test_optimize_empty_code_fts_index_preserves_params() { + async fn test_optimize_empty_code_fts_index_preserves_params(#[case] block_size: usize) { let dir = TempStrDir::default(); let schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::Int32, false), @@ -5028,6 +5031,8 @@ mod tests { let params = InvertedIndexParams::default() .analyzer("code") .unwrap() + .block_size(block_size) + .unwrap() .split_identifiers(true) .preserve_original(false); dataset @@ -5039,13 +5044,26 @@ mod tests { let indices = dataset.load_indices().await.unwrap(); assert_eq!(indices.len(), 1); - assert_eq!(indices[0].index_version, INVERTED_INDEX_VERSION_V4 as i32); + assert_eq!(indices[0].index_version, INVERTED_INDEX_VERSION_V3 as i32); dataset.optimize_indices(&Default::default()).await.unwrap(); let indices = dataset.load_indices().await.unwrap(); assert_eq!(indices.len(), 1); - assert_eq!(indices[0].index_version, INVERTED_INDEX_VERSION_V4 as i32); + assert_eq!(indices[0].index_version, INVERTED_INDEX_VERSION_V3 as i32); + let details = indices[0] + .index_details + .as_ref() + .expect("optimized FTS index should retain index details") + .to_msg::() + .unwrap(); + assert_eq!(details.base_tokenizer.as_deref(), Some("code")); + assert_eq!(details.block_size, Some(block_size as u32)); + let code_config = details + .code_config + .expect("optimized code FTS index should retain code configuration"); + assert!(code_config.split_identifiers); + assert_eq!(code_config.preserve_original, Some(false)); let stats: serde_json::Value = serde_json::from_str(&dataset.index_statistics("code_idx").await.unwrap()).unwrap();