feat(index): add FTS document granularity

docs/src/guide/migration.md

@@ -10,7 +10,8 @@ migrate.
 
 * Unless overridden, newly created FTS indexes use format v2. The code analyzer
   and `block_size=256` require format v3, so readers must support v3 before an
-  index using either option is created.
+  index using either option is created. `document_granularity="list_element"`
+  also requires v3 reader capability, independently of the posting format.
 
 * To keep new indexes readable by nodes that support at most format v1 or v2,
   set `format_version` in the index creation parameters, or set

java/lance-jni/src/blocking_dataset.rs

@@ -3402,6 +3402,7 @@ fn inner_describe_indices<'local>(
             for_column: for_column.as_deref(),
             has_name: has_name.as_deref(),
             must_support_fts,
+            fts_document_granularity: None,
             must_support_exact_equality,
         })
     })?;
@@ -3579,6 +3580,7 @@ fn inner_get_zonemap_stats<'local>(
                     for_column: Some(&column_name),
                     has_name: None,
                     must_support_fts: false,
+                    fts_document_granularity: None,
                     must_support_exact_equality: false,
                 }))
                 .await

java/lance-jni/src/blocking_scanner.rs

@@ -18,10 +18,13 @@ use lance::dataset::scanner::{
     ExecutionSummaryCounts, Scanner,
 };
 use lance_index::scalar::FullTextSearchQuery;
-use lance_index::scalar::inverted::query::{
-    BooleanQuery as FtsBooleanQuery, BoostQuery as FtsBoostQuery, FtsQuery,
-    MatchQuery as FtsMatchQuery, MultiMatchQuery as FtsMultiMatchQuery, Occur as FtsOccur,
-    PhraseQuery as FtsPhraseQuery,
+use lance_index::scalar::inverted::{
+    DocumentGranularity,
+    query::{
+        BooleanQuery as FtsBooleanQuery, BoostQuery as FtsBoostQuery, FtsQuery,
+        MatchQuery as FtsMatchQuery, MultiMatchQuery as FtsMultiMatchQuery, Occur as FtsOccur,
+        PhraseQuery as FtsPhraseQuery,
+    },
 };
 use lance_io::ffi::to_ffi_arrow_array_stream;
 use lance_linalg::distance::DistanceType;
@@ -114,6 +117,7 @@ pub(crate) fn build_full_text_search_query<'a>(
             let max_expansions = env.get_int_as_usize_from_method(&java_obj, "getMaxExpansions")?;
             let operator = env.get_fts_operator_from_method(&java_obj)?;
             let prefix_length = env.get_u32_from_method(&java_obj, "getPrefixLength")?;
+            let document_granularity = get_document_granularity(env, &java_obj)?;
 
             let mut query = FtsMatchQuery::new(query_text);
             query = query.with_column(Some(column));
@@ -123,17 +127,24 @@ pub(crate) fn build_full_text_search_query<'a>(
                 .with_max_expansions(max_expansions)
                 .with_operator(operator)
                 .with_prefix_length(prefix_length);
+            if let Some(document_granularity) = document_granularity {
+                query = query.with_document_granularity(document_granularity);
+            }
 
             Ok(FtsQuery::Match(query))
         }
         "MATCH_PHRASE" => {
             let query_text = env.get_string_from_method(&java_obj, "getQueryText")?;
             let column = env.get_string_from_method(&java_obj, "getColumn")?;
             let slop = env.get_u32_from_method(&java_obj, "getSlop")?;
+            let document_granularity = get_document_granularity(env, &java_obj)?;
 
             let mut query = FtsPhraseQuery::new(query_text);
             query = query.with_column(Some(column));
             query = query.with_slop(slop);
+            if let Some(document_granularity) = document_granularity {
+                query = query.with_document_granularity(document_granularity);
+            }
 
             Ok(FtsQuery::Phrase(query))
         }
@@ -229,6 +240,20 @@ pub(crate) fn build_full_text_search_query<'a>(
     }
 }
 
+fn get_document_granularity(
+    env: &mut JNIEnv<'_>,
+    java_obj: &JObject,
+) -> Result<Option<DocumentGranularity>> {
+    env.get_optional_from_method(
+        java_obj,
+        "getDocumentGranularity",
+        |env, granularity_obj| {
+            let value = env.get_string_from_method(&granularity_obj, "toRustString")?;
+            DocumentGranularity::try_from(value.as_str()).map_err(Error::from)
+        },
+    )
+}
+
 /// Scanner options passed from JNI - shared between blocking and async scanners
 pub(crate) struct ScannerOptions<'a> {
     pub fragment_ids_obj: JObject<'a>,

java/src/main/java/org/lance/DocumentGranularity.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.lance;
+
+/** The unit treated as one full-text-search document. */
+public enum DocumentGranularity {
+  /** All text selected from one dataset row belongs to one document. */
+  ROW("row"),
+
+  /** Each element of the deepest list on the field path is one document. */
+  LIST_ELEMENT("list_element");
+
+  private final String rustString;
+
+  DocumentGranularity(String rustString) {
+    this.rustString = rustString;
+  }
+
+  /** Return the stable value understood by the Rust API and serialized index parameters. */
+  public String toRustString() {
+    return rustString;
+  }
+}

java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java

@@ -13,6 +13,7 @@
  */
 package org.lance.index.scalar;
 
+import org.lance.DocumentGranularity;
 import org.lance.util.JsonUtils;
 
 import com.google.common.base.Preconditions;
@@ -56,6 +57,7 @@ public static final class Builder {
     private Integer blockSize = 128;
     private Boolean skipMerge;
     private Integer formatVersion;
+    private DocumentGranularity documentGranularity = DocumentGranularity.ROW;
 
     /**
      * Configure the base tokenizer.
@@ -282,6 +284,21 @@ public Builder formatVersion(int formatVersion) {
       return this;
     }
 
+    /**
+     * Configure the unit treated as one FTS document.
+     *
+     * <p>{@link DocumentGranularity#LIST_ELEMENT} uses each element of the deepest list on the
+     * indexed field path as one document. The default is {@link DocumentGranularity#ROW}.
+     *
+     * @param documentGranularity document boundary semantics
+     * @return this builder
+     */
+    public Builder documentGranularity(DocumentGranularity documentGranularity) {
+      this.documentGranularity =
+          Objects.requireNonNull(documentGranularity, "documentGranularity must not be null");
+      return this;
+    }
+
     /** Build a {@link ScalarIndexParams} instance for an inverted index. */
     public ScalarIndexParams build() {
       if (formatVersion != null) {
@@ -342,6 +359,7 @@ public ScalarIndexParams build() {
       if (formatVersion != null) {
         params.put("format_version", formatVersion);
       }
+      params.put("document_granularity", documentGranularity.toRustString());
 
       String json = JsonUtils.toJson(params);
       return ScalarIndexParams.create(INDEX_TYPE, json);