refactor(fts): separate document identity from row addresses

docs/src/format/index/scalar/fts.md

@@ -34,6 +34,15 @@ An FTS index may contain multiple partitions. Each partition has its own set of
 | `_rowid`      | UInt64 | false    | Document row ID                  |
 | `_num_tokens` | UInt32 | false    | Number of tokens in the document |
 
+Partitioned `docs.lance` files may include the optional schema metadata key
+`total_tokens`. Its decimal `UInt64` value is the sum of `_num_tokens` in that
+file. `_num_tokens` remains the canonical per-document data. Readers use the
+metadata value to construct exact corpus statistics without scanning the column;
+when the key is absent, they compute the sum from `_num_tokens`. Writers produce
+the key from the same document table in the same file commit. A present value
+that cannot be parsed, or that differs when `_num_tokens` is subsequently
+loaded, is file corruption.
+
 ### FTS List File Schema
 
 | Column                 | Type                    | Nullable | Description                                                      |

rust/lance-index/src/scalar/inverted.rs

@@ -3,12 +3,12 @@
 
 pub mod builder;
 mod cache_codec;
+mod documents;
 mod encoding;
 mod impact;
 mod index;
 mod iter;
 pub mod json;
-mod lazy_docset;
 pub mod parser;
 pub mod query;
 mod scorer;

rust/lance-index/src/scalar/inverted/builder.rs

@@ -1200,7 +1200,12 @@ impl InnerBuilder {
         let batch = docs.to_batch()?;
         let mut writer = store.new_index_file(path, batch.schema()).await?;
         writer.write_record_batch(batch).await?;
-        writer.finish().await
+        writer
+            .finish_with_metadata(HashMap::from([(
+                super::documents::TOTAL_TOKENS_KEY.to_owned(),
+                docs.total_tokens_num().to_string(),
+            )]))
+            .await
     }
 }