fix(fts): use v3 as the capability gate

.github/workflows/rust.yml

@@ -85,7 +85,7 @@ jobs:
 
   linux-build:
     runs-on: "ubuntu-24.04-8x"
-    timeout-minutes: 60
+    timeout-minutes: 75
     env:
       # Need up-to-date compilers for kernels
       CC: clang

docs/src/guide/migration.md

@@ -8,29 +8,18 @@ migrate.
 
 ## 9.0.0
 
-* Newly created FTS / inverted indexes now default to format v4 instead of v1.
-  Format selection follows this priority: an explicit index creation parameter
-  `format_version`, then the `LANCE_FTS_FORMAT_VERSION` environment variable,
-  then v4 when neither is set. This preserves the environment variable as a
-  compatibility fallback for deployments that use it as a global rollout
-  switch. An invalid environment variable value fails index creation instead of
-  silently selecting v4; an explicit `format_version` takes precedence even
-  when the environment variable is invalid.
-
-* This affects users who create FTS / inverted indexes and need those indexes to
-  be readable by older Lance versions, or who depend on the v1 index layout. In
-  those cases, pass `format_version=1` when creating the index, or set
-  `LANCE_FTS_FORMAT_VERSION=1` for a process-wide rollout. When neither setting
-  is present, newly created indexes use v4, and older Lance readers may not be
-  able to read them.
-
-  ```python
-  dataset.create_scalar_index("text", "INVERTED", format_version=1)
-  ```
-
-* Existing v1 FTS indexes remain queryable. Operations that maintain an existing
-  v1 index, including append, incremental indexing, optimize, and mem-wal
-  maintained-index flush, should continue preserving the v1 format.
+* Unless overridden, newly created FTS indexes use format v2. The code analyzer
+  and `block_size=256` require format v3, so readers must support v3 before an
+  index using either option is created.
+
+* To keep new indexes readable by nodes that support at most format v1 or v2,
+  set `format_version` in the index creation parameters, or set
+  `LANCE_FTS_FORMAT_VERSION` for a rollout-wide override. Formats v1 and v2
+  require the text analyzer and `block_size=128`.
+
+* Operations that maintain an existing FTS index, including append, incremental
+  indexing, optimize, and mem-wal maintained-index flush, preserve its format
+  version.
 
 ## 7.2.0
 

java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java

@@ -67,6 +67,7 @@ public static final class Builder {
      *   <li>{@code "whitespace"}: splits tokens on whitespace
      *   <li>{@code "raw"}: no tokenization
      *   <li>{@code "ngram"}: N-Gram tokenizer
+     *   <li>{@code "code"}: code-aware tokenizer
      *   <li>{@code "icu"}: ICU dictionary-based Unicode word segmentation
      *   <li>{@code "icu/split"}: ICU segmentation with simple-style delimiter splitting
      *   <li>{@code "lindera/*"}: Lindera tokenizer
@@ -233,8 +234,8 @@ public Builder prefixOnly(boolean prefixOnly) {
      * <p>Supported values are {@code 128} and {@code 256}. New indexes default to {@code 128} when
      * this is not set.
      *
-     * <p>{@code blockSize = 256} is experimental and may introduce breaking changes. Use {@code
-     * 128} when stable compatibility with the legacy posting layout is required.
+     * <p>{@code blockSize = 256} requires FTS format v3. Format v3 also supports the default {@code
+     * blockSize = 128}.
      *
      * @param blockSize posting block size
      * @return this builder
@@ -264,16 +265,18 @@ public Builder skipMerge(boolean skipMerge) {
     /**
      * Configure the on-disk FTS format version to write when creating a new index.
      *
-     * <p>If unset, Lance uses {@code LANCE_FTS_FORMAT_VERSION} when present and otherwise writes
-     * v4. {@code formatVersion = 3} is experimental and is only valid with {@code blockSize = 256}.
+     * <p>If unset, Lance uses {@code LANCE_FTS_FORMAT_VERSION} when present and otherwise selects
+     * v3 for the code analyzer or {@code blockSize = 256}, and v2 for other indexes. Format v3
+     * supports both posting block sizes. Formats v1 and v2 support only {@code blockSize = 128} and
+     * cannot be used with the code analyzer.
      *
-     * @param formatVersion FTS format version, must be 1, 2, 3, or 4
+     * @param formatVersion FTS format version, must be 1, 2, or 3
      * @return this builder
      * @throws IllegalArgumentException
      */
     public Builder formatVersion(int formatVersion) {
-      if (formatVersion != 1 && formatVersion != 2 && formatVersion != 3 && formatVersion != 4) {
-        throw new IllegalArgumentException("formatVersion must be 1, 2, 3, or 4");
+      if (formatVersion != 1 && formatVersion != 2 && formatVersion != 3) {
+        throw new IllegalArgumentException("formatVersion must be 1, 2, or 3");
       }
       this.formatVersion = formatVersion;
       return this;
@@ -283,10 +286,10 @@ public Builder formatVersion(int formatVersion) {
     public ScalarIndexParams build() {
       if (formatVersion != null) {
         Preconditions.checkArgument(
-            formatVersion == 4
-                || (blockSize == 256 && formatVersion == 3)
-                || (blockSize == 128 && formatVersion != 3),
-            "formatVersion 3 requires blockSize 256, and legacy formats require blockSize 128");
+            formatVersion == 3 || blockSize == 128, "formatVersion 1 and 2 require blockSize 128");
+        Preconditions.checkArgument(
+            !"code".equals(baseTokenizer) || formatVersion == 3,
+            "baseTokenizer 'code' requires formatVersion 3");
       }
       Map<String, Object> params = new HashMap<>();
       if (baseTokenizer != null) {

java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java

@@ -60,30 +60,35 @@ void invalidBlockSizeIsRejected() {
   }
 
   @Test
-  void formatVersionThreeRequiresBlockSize256() {
-    ScalarIndexParams params =
-        InvertedIndexParams.builder().blockSize(256).formatVersion(3).build();
-
-    Map<String, Object> json = JsonUtils.fromJson(params.getJsonParams().orElseThrow());
-    assertEquals(256, ((Number) json.get("block_size")).intValue());
-    assertEquals(3, ((Number) json.get("format_version")).intValue());
+  void formatVersionThreeSupportsBothBlockSizes() {
+    for (int blockSize : new int[] {128, 256}) {
+      ScalarIndexParams params =
+          InvertedIndexParams.builder().blockSize(blockSize).formatVersion(3).build();
+      Map<String, Object> json = JsonUtils.fromJson(params.getJsonParams().orElseThrow());
+      assertEquals(blockSize, ((Number) json.get("block_size")).intValue());
+      assertEquals(3, ((Number) json.get("format_version")).intValue());
+    }
 
-    assertThrows(
-        IllegalArgumentException.class,
-        () -> InvertedIndexParams.builder().formatVersion(3).build());
     assertThrows(
         IllegalArgumentException.class,
         () -> InvertedIndexParams.builder().blockSize(256).formatVersion(2).build());
   }
 
   @Test
-  void formatVersionFourSupportsBothBlockSizes() {
-    for (int blockSize : new int[] {128, 256}) {
-      ScalarIndexParams params =
-          InvertedIndexParams.builder().blockSize(blockSize).formatVersion(4).build();
-      Map<String, Object> json = JsonUtils.fromJson(params.getJsonParams().orElseThrow());
-      assertEquals(blockSize, ((Number) json.get("block_size")).intValue());
-      assertEquals(4, ((Number) json.get("format_version")).intValue());
-    }
+  void formatVersionFourIsRejected() {
+    assertThrows(
+        IllegalArgumentException.class, () -> InvertedIndexParams.builder().formatVersion(4));
+  }
+
+  @Test
+  void codeAnalyzerRequiresFormatVersionThreeWhenExplicit() {
+    ScalarIndexParams params =
+        InvertedIndexParams.builder().baseTokenizer("code").formatVersion(3).build();
+    Map<String, Object> json = JsonUtils.fromJson(params.getJsonParams().orElseThrow());
+    assertEquals(3, ((Number) json.get("format_version")).intValue());
+
+    assertThrows(
+        IllegalArgumentException.class,
+        () -> InvertedIndexParams.builder().baseTokenizer("code").formatVersion(2).build());
   }
 }

python/python/lance/dataset.py

@@ -3370,11 +3370,10 @@ def create_scalar_index(
         format_version: int or str, optional
             This is for the ``INVERTED`` / ``FTS`` index. Explicit on-disk FTS
             format version to write when creating a new index. Accepts ``1``,
-            ``2``, ``3``, ``4``, ``"v1"``, ``"v2"``, ``"v3"``, or ``"v4"``.
+            ``2``, ``3``, ``"v1"``, ``"v2"``, or ``"v3"``.
             If unset, Lance uses ``LANCE_FTS_FORMAT_VERSION`` when present and
-            otherwise writes v4.
-            ``format_version=3`` is experimental and is only valid with
-            ``block_size=256``.
+            otherwise writes v2 for text analysis with ``block_size=128`` and
+            v3 for code analysis or ``block_size=256``.
 
         with_position: bool, default False
             This is for the ``INVERTED`` index. If True, the index will store the

python/python/tests/test_scalar_index.py

@@ -809,10 +809,17 @@ def test_full_text_search(dataset, with_position, base_tokenizer):
         )
 
 
-def test_code_analyzer_does_not_split_identifiers_by_default(tmp_path):
+@pytest.mark.parametrize("block_size", [128, 256])
+def test_code_analyzer_does_not_split_identifiers_by_default(tmp_path, block_size):
     table = pa.table({"code": ["GetUserName", "GetUserEmail", "user"]})
     ds = lance.write_dataset(table, tmp_path)
-    ds.create_scalar_index("code", index_type="INVERTED", analyzer="code")
+    ds.create_scalar_index(
+        "code",
+        index_type="INVERTED",
+        analyzer="code",
+        block_size=block_size,
+    )
+    assert ds.describe_indices()[0].segments[0].index_version == 3
 
     results = ds.to_table(
         columns=["code"],
@@ -927,6 +934,19 @@ def test_code_analyzer_flags_require_code_analyzer(tmp_path):
         )
 
 
+def test_code_analyzer_requires_fts_v3(tmp_path):
+    table = pa.table({"code": ["getUserName"]})
+    ds = lance.write_dataset(table, tmp_path)
+
+    with pytest.raises(ValueError, match="requires FTS format_version=3"):
+        ds.create_scalar_index(
+            "code",
+            index_type="INVERTED",
+            analyzer="code",
+            format_version=2,
+        )
+
+
 def test_code_analyzer_complex_code_constructs(tmp_path):
     table = pa.table(
         {
@@ -1273,7 +1293,7 @@ def test_create_scalar_index_fts_block_size(dataset):
     )
     indices = dataset.describe_indices()
     doc_index = next(index for index in indices if index.name == "doc_idx")
-    assert doc_index.segments[0].index_version == 4
+    assert doc_index.segments[0].index_version == 3
 
     row = dataset.take(indices=[0], columns=["doc"])
     query = row.column(0)[0].as_py().split(" ")[0]
@@ -5392,7 +5412,7 @@ def test_json_inverted_match_query(tmp_path):
 
 @pytest.mark.parametrize(
     ("format_version", "expected_format_version"),
-    [(1, 1), (2, 2), (4, 4), ("v1", 1), ("v2", 2), ("v4", 4)],
+    [(1, 1), (2, 2), (3, 3), ("v1", 1), ("v2", 2), ("v3", 3)],
 )
 def test_describe_indices(tmp_path, format_version, expected_format_version):
     data = pa.table(
@@ -5568,8 +5588,8 @@ def _run_fts_format_creation_probe(
     [
         ("1", {}, 1),
         ("2", {}, 2),
+        ("3", {}, 3),
         ("3", {"block_size": 256}, 3),
-        ("4", {}, 4),
     ],
 )
 def test_create_inverted_index_uses_env_format_version(
@@ -5596,8 +5616,8 @@ def test_create_inverted_index_explicit_format_version_overrides_env(tmp_path):
     assert result.returncode == 0, result.stderr
 
 
-def test_create_inverted_index_defaults_to_v4_without_env(tmp_path):
-    result = _run_fts_format_creation_probe(tmp_path, None, expected_format_version=4)
+def test_create_text_inverted_index_defaults_to_v2_without_env(tmp_path):
+    result = _run_fts_format_creation_probe(tmp_path, None, expected_format_version=2)
 
     assert result.returncode == 0, result.stderr
 
@@ -5617,8 +5637,8 @@ def test_create_inverted_index_rejects_invalid_format_version(tmp_path):
     with pytest.raises(ValueError, match="unsupported FTS format version"):
         ds.create_scalar_index("text", index_type="INVERTED", format_version="v5")
 
-    with pytest.raises(ValueError, match="format_version=3"):
-        ds.create_scalar_index("text", index_type="INVERTED", format_version="v3")
+    with pytest.raises(ValueError, match="unsupported FTS format version"):
+        ds.create_scalar_index("text", index_type="INVERTED", format_version="v4")
 
 
 def test_vector_filter_fts_search(tmp_path):

python/src/dataset.rs

@@ -2583,7 +2583,7 @@ impl Dataset {
                             value.to_string()
                         } else {
                             return Err(PyValueError::new_err(
-                                "format_version must be 1, 2, 3, 4, 'v1', 'v2', 'v3', or 'v4'",
+                                "format_version must be 1, 2, 3, 'v1', 'v2', or 'v3'",
                             ));
                         };
                         let format_version = value

rust/lance-index/src/scalar/inverted.rs

@@ -268,7 +268,7 @@ impl ScalarIndexPlugin for InvertedIndexPlugin {
     }
 
     fn version(&self) -> u32 {
-        INVERTED_INDEX_VERSION_V4
+        INVERTED_INDEX_VERSION_V3
     }
 
     fn new_query_parser(
@@ -320,9 +320,9 @@ mod tests {
     use crate::scalar::{BuiltinIndexType, ScalarIndexParams};
 
     #[test]
-    fn test_plugin_version_tracks_current_index_version() {
+    fn test_plugin_version_tracks_v3_capability_gate() {
         let plugin = InvertedIndexPlugin;
-        assert_eq!(plugin.version(), INVERTED_INDEX_VERSION_V4);
+        assert_eq!(plugin.version(), INVERTED_INDEX_VERSION_V3);
     }
 
     #[test]

rust/lance-index/src/scalar/inverted/builder.rs

@@ -1820,16 +1820,16 @@ pub(crate) fn inverted_list_schema_for_version_with_block_size_and_impacts(
         InvertedListFormatVersion::V1 => {
             inverted_list_schema_v1(with_position, block_size, with_impacts)
         }
-        InvertedListFormatVersion::V2
-        | InvertedListFormatVersion::V3
-        | InvertedListFormatVersion::V4 => inverted_list_schema_with_tail_codec_and_position_codec(
-            with_position,
-            format_version,
-            PostingTailCodec::VarintDelta,
-            Some(PositionStreamCodec::PackedDelta),
-            block_size,
-            with_impacts,
-        ),
+        InvertedListFormatVersion::V2 | InvertedListFormatVersion::V3 => {
+            inverted_list_schema_with_tail_codec_and_position_codec(
+                with_position,
+                format_version,
+                PostingTailCodec::VarintDelta,
+                Some(PositionStreamCodec::PackedDelta),
+                block_size,
+                with_impacts,
+            )
+        }
     }
 }
 
@@ -2927,7 +2927,7 @@ mod tests {
         assert_eq!(written_partitions, remapped_partitions);
         assert_eq!(
             parse_format_version_from_metadata(metadata)?,
-            InvertedListFormatVersion::V4
+            InvertedListFormatVersion::V2
         );
 
         for (new_id, old_id) in expected_partitions.iter().enumerate() {
@@ -3452,6 +3452,39 @@ mod tests {
         assert_eq!(builder.posting_tail_codec, PostingTailCodec::VarintDelta);
     }
 
+    #[test]
+    fn test_v3_128_reuses_v2_physical_layout() {
+        for with_position in [false, true] {
+            for with_impacts in [false, true] {
+                let v2 = inverted_list_schema_for_version_with_block_size_and_impacts(
+                    with_position,
+                    InvertedListFormatVersion::V2,
+                    LEGACY_BLOCK_SIZE,
+                    with_impacts,
+                );
+                let v3 = inverted_list_schema_for_version_with_block_size_and_impacts(
+                    with_position,
+                    InvertedListFormatVersion::V3,
+                    LEGACY_BLOCK_SIZE,
+                    with_impacts,
+                );
+
+                assert_eq!(v2.fields(), v3.fields());
+                let mut v2_metadata = v2.metadata.clone();
+                let mut v3_metadata = v3.metadata.clone();
+                assert_eq!(
+                    v2_metadata.remove(FTS_FORMAT_VERSION_KEY).as_deref(),
+                    Some("2")
+                );
+                assert_eq!(
+                    v3_metadata.remove(FTS_FORMAT_VERSION_KEY).as_deref(),
+                    Some("3")
+                );
+                assert_eq!(v2_metadata, v3_metadata);
+            }
+        }
+    }
+
     #[tokio::test]
     async fn test_inverted_index_without_positions_tracks_frequency() -> Result<()> {
         let index_dir = TempDir::default();

rust/lance-index/src/scalar/inverted/encoding.rs

@@ -128,8 +128,8 @@ pub fn compress_posting_list_with_tail_codec_and_block_size<'a>(
 }
 
 /// Byte length of the block-max-score prefix on posting blocks. 128-doc
-/// blocks store a per-block max score, patched in at build time; 256-doc
-/// (V3) blocks always carry impact skip data, which supersedes it, so they
+/// blocks store a per-block max score, patched in at build time; 256-document
+/// blocks always carry impact skip data, which supersedes it, so they
 /// store none.
 #[inline]
 pub fn posting_block_score_prefix_len(block_size: usize) -> usize {
@@ -167,7 +167,7 @@ fn encode_posting_block_payload(
             compress_sorted_block_with::<BitPacker4x>(doc_ids, &mut buffer, block)?;
             compress_block_with::<BitPacker4x>(frequencies, &mut buffer, block)?;
         }
-        // 256-doc blocks (format V3) store frequencies with patched FOR:
+        // 256-document blocks store frequencies with patched FOR:
         // outliers no longer widen the whole block, which matters because one
         // large tf per block otherwise doubles the frequency payload.
         BitPacker8x::BLOCK_LEN => {