feat(index): add FTS document granularity

java/lance-jni/src/blocking_dataset.rs

@@ -3402,6 +3402,7 @@ fn inner_describe_indices<'local>(
             for_column: for_column.as_deref(),
             has_name: has_name.as_deref(),
             must_support_fts,
+            fts_document_granularity: None,
             must_support_exact_equality,
         })
     })?;
@@ -3579,6 +3580,7 @@ fn inner_get_zonemap_stats<'local>(
                     for_column: Some(&column_name),
                     has_name: None,
                     must_support_fts: false,
+                    fts_document_granularity: None,
                     must_support_exact_equality: false,
                 }))
                 .await

java/lance-jni/src/blocking_scanner.rs

@@ -18,10 +18,13 @@ use lance::dataset::scanner::{
     ExecutionSummaryCounts, Scanner,
 };
 use lance_index::scalar::FullTextSearchQuery;
-use lance_index::scalar::inverted::query::{
-    BooleanQuery as FtsBooleanQuery, BoostQuery as FtsBoostQuery, FtsQuery,
-    MatchQuery as FtsMatchQuery, MultiMatchQuery as FtsMultiMatchQuery, Occur as FtsOccur,
-    PhraseQuery as FtsPhraseQuery,
+use lance_index::scalar::inverted::{
+    DocumentGranularity,
+    query::{
+        BooleanQuery as FtsBooleanQuery, BoostQuery as FtsBoostQuery, FtsQuery,
+        MatchQuery as FtsMatchQuery, MultiMatchQuery as FtsMultiMatchQuery, Occur as FtsOccur,
+        PhraseQuery as FtsPhraseQuery,
+    },
 };
 use lance_io::ffi::to_ffi_arrow_array_stream;
 use lance_linalg::distance::DistanceType;
@@ -114,6 +117,7 @@ pub(crate) fn build_full_text_search_query<'a>(
             let max_expansions = env.get_int_as_usize_from_method(&java_obj, "getMaxExpansions")?;
             let operator = env.get_fts_operator_from_method(&java_obj)?;
             let prefix_length = env.get_u32_from_method(&java_obj, "getPrefixLength")?;
+            let document_granularity = get_document_granularity(env, &java_obj)?;
 
             let mut query = FtsMatchQuery::new(query_text);
             query = query.with_column(Some(column));
@@ -123,17 +127,24 @@ pub(crate) fn build_full_text_search_query<'a>(
                 .with_max_expansions(max_expansions)
                 .with_operator(operator)
                 .with_prefix_length(prefix_length);
+            if let Some(document_granularity) = document_granularity {
+                query = query.with_document_granularity(document_granularity);
+            }
 
             Ok(FtsQuery::Match(query))
         }
         "MATCH_PHRASE" => {
             let query_text = env.get_string_from_method(&java_obj, "getQueryText")?;
             let column = env.get_string_from_method(&java_obj, "getColumn")?;
             let slop = env.get_u32_from_method(&java_obj, "getSlop")?;
+            let document_granularity = get_document_granularity(env, &java_obj)?;
 
             let mut query = FtsPhraseQuery::new(query_text);
             query = query.with_column(Some(column));
             query = query.with_slop(slop);
+            if let Some(document_granularity) = document_granularity {
+                query = query.with_document_granularity(document_granularity);
+            }
 
             Ok(FtsQuery::Phrase(query))
         }
@@ -229,6 +240,20 @@ pub(crate) fn build_full_text_search_query<'a>(
     }
 }
 
+fn get_document_granularity(
+    env: &mut JNIEnv<'_>,
+    java_obj: &JObject,
+) -> Result<Option<DocumentGranularity>> {
+    env.get_optional_from_method(
+        java_obj,
+        "getDocumentGranularity",
+        |env, granularity_obj| {
+            let value = env.get_string_from_method(&granularity_obj, "toRustString")?;
+            DocumentGranularity::try_from(value.as_str()).map_err(Error::from)
+        },
+    )
+}
+
 /// Scanner options passed from JNI - shared between blocking and async scanners
 pub(crate) struct ScannerOptions<'a> {
     pub fragment_ids_obj: JObject<'a>,

java/src/main/java/org/lance/DocumentGranularity.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.lance;
+
+/** The unit treated as one full-text-search document. */
+public enum DocumentGranularity {
+  /** All text selected from one dataset row belongs to one document. */
+  ROW("row"),
+
+  /** Each element of the deepest list on the field path is one document. */
+  LIST_ELEMENT("list_element");
+
+  private final String rustString;
+
+  DocumentGranularity(String rustString) {
+    this.rustString = rustString;
+  }
+
+  /** Return the stable value understood by the Rust API and serialized index parameters. */
+  public String toRustString() {
+    return rustString;
+  }
+}

java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java

@@ -13,6 +13,7 @@
  */
 package org.lance.index.scalar;
 
+import org.lance.DocumentGranularity;
 import org.lance.util.JsonUtils;
 
 import com.google.common.base.Preconditions;
@@ -56,6 +57,7 @@ public static final class Builder {
     private Integer blockSize = 128;
     private Boolean skipMerge;
     private Integer formatVersion;
+    private DocumentGranularity documentGranularity = DocumentGranularity.ROW;
 
     /**
      * Configure the base tokenizer.
@@ -279,6 +281,21 @@ public Builder formatVersion(int formatVersion) {
       return this;
     }
 
+    /**
+     * Configure the unit treated as one FTS document.
+     *
+     * <p>{@link DocumentGranularity#LIST_ELEMENT} uses each element of the deepest list on the
+     * indexed field path as one document. The default is {@link DocumentGranularity#ROW}.
+     *
+     * @param documentGranularity document boundary semantics
+     * @return this builder
+     */
+    public Builder documentGranularity(DocumentGranularity documentGranularity) {
+      this.documentGranularity =
+          Objects.requireNonNull(documentGranularity, "documentGranularity must not be null");
+      return this;
+    }
+
     /** Build a {@link ScalarIndexParams} instance for an inverted index. */
     public ScalarIndexParams build() {
       if (formatVersion != null) {
@@ -339,6 +356,7 @@ public ScalarIndexParams build() {
       if (formatVersion != null) {
         params.put("format_version", formatVersion);
       }
+      params.put("document_granularity", documentGranularity.toRustString());
 
       String json = JsonUtils.toJson(params);
       return ScalarIndexParams.create(INDEX_TYPE, json);

java/src/main/java/org/lance/ipc/FullTextQuery.java

@@ -13,6 +13,8 @@
  */
 package org.lance.ipc;
 
+import org.lance.DocumentGranularity;
+
 import com.google.common.base.MoreObjects;
 import org.apache.arrow.util.Preconditions;
 
@@ -21,7 +23,13 @@
 import java.util.Objects;
 import java.util.Optional;
 
-/** Base type for full text search queries used by Lance scanner. */
+/**
+ * Base type for full text search queries used by Lance scanner.
+ *
+ * <p>Match and phrase overloads without a {@link DocumentGranularity} infer the unique indexed
+ * granularity for the field. They use row documents when no index exists and fail as ambiguous when
+ * row and list-element indexes coexist.
+ */
 public abstract class FullTextQuery {
   public enum Type {
     MATCH,
@@ -63,7 +71,14 @@ public FullTextQuery getQuery() {
   public abstract Type getType();
 
   public static FullTextQuery match(String queryText, String column) {
-    return match(queryText, column, 1.0f, Optional.empty(), 50, Operator.OR, 0);
+    return new MatchQuery(
+        queryText, column, 1.0f, Optional.empty(), 50, Operator.OR, 0, Optional.empty());
+  }
+
+  public static FullTextQuery match(
+      String queryText, String column, DocumentGranularity documentGranularity) {
+    return match(
+        queryText, column, 1.0f, Optional.empty(), 50, Operator.OR, 0, documentGranularity);
   }
 
   public static FullTextQuery match(
@@ -75,15 +90,58 @@ public static FullTextQuery match(
       Operator operator,
       int prefixLength) {
     return new MatchQuery(
-        queryText, column, boost, fuzziness, maxExpansions, operator, prefixLength);
+        queryText,
+        column,
+        boost,
+        fuzziness,
+        maxExpansions,
+        operator,
+        prefixLength,
+        Optional.empty());
+  }
+
+  public static FullTextQuery match(
+      String queryText,
+      String column,
+      float boost,
+      Optional<Integer> fuzziness,
+      int maxExpansions,
+      Operator operator,
+      int prefixLength,
+      DocumentGranularity documentGranularity) {
+    return new MatchQuery(
+        queryText,
+        column,
+        boost,
+        fuzziness,
+        maxExpansions,
+        operator,
+        prefixLength,
+        Optional.of(
+            Objects.requireNonNull(documentGranularity, "documentGranularity must not be null")));
   }
 
   public static FullTextQuery phrase(String queryText, String column) {
-    return phrase(queryText, column, 0);
+    return new PhraseQuery(queryText, column, 0, Optional.empty());
+  }
+
+  public static FullTextQuery phrase(
+      String queryText, String column, DocumentGranularity documentGranularity) {
+    return phrase(queryText, column, 0, documentGranularity);
   }
 
   public static FullTextQuery phrase(String queryText, String column, int slop) {
-    return new PhraseQuery(queryText, column, slop);
+    return new PhraseQuery(queryText, column, slop, Optional.empty());
+  }
+
+  public static FullTextQuery phrase(
+      String queryText, String column, int slop, DocumentGranularity documentGranularity) {
+    return new PhraseQuery(
+        queryText,
+        column,
+        slop,
+        Optional.of(
+            Objects.requireNonNull(documentGranularity, "documentGranularity must not be null")));
   }
 
   public static FullTextQuery multiMatch(String queryText, List<String> columns) {
@@ -117,6 +175,7 @@ public static final class MatchQuery extends FullTextQuery {
     private final int maxExpansions;
     private final Operator operator;
     private final int prefixLength;
+    private final Optional<DocumentGranularity> documentGranularity;
 
     MatchQuery(
         String queryText,
@@ -125,7 +184,8 @@ public static final class MatchQuery extends FullTextQuery {
         Optional<Integer> fuzziness,
         int maxExpansions,
         Operator operator,
-        int prefixLength) {
+        int prefixLength,
+        Optional<DocumentGranularity> documentGranularity) {
       Preconditions.checkArgument(
           queryText != null && !queryText.isEmpty(), "queryText must not be null or empty");
       Preconditions.checkArgument(
@@ -140,6 +200,7 @@ public static final class MatchQuery extends FullTextQuery {
       this.maxExpansions = maxExpansions;
       this.operator = operator == null ? Operator.OR : operator;
       this.prefixLength = prefixLength;
+      this.documentGranularity = Objects.requireNonNull(documentGranularity);
     }
 
     @Override
@@ -175,6 +236,11 @@ public int getPrefixLength() {
       return prefixLength;
     }
 
+    /** Returns the explicit granularity, or empty when query planning should infer it. */
+    public Optional<DocumentGranularity> getDocumentGranularity() {
+      return documentGranularity;
+    }
+
     @Override
     public boolean equals(Object o) {
       if (this == o) return true;
@@ -184,6 +250,7 @@ public boolean equals(Object o) {
           && maxExpansions == other.maxExpansions
           && prefixLength == other.prefixLength
           && operator == other.operator
+          && Objects.equals(documentGranularity, other.documentGranularity)
           && Objects.equals(queryText, other.queryText)
           && Objects.equals(column, other.column)
           && Objects.equals(fuzziness, other.fuzziness);
@@ -192,7 +259,14 @@ public boolean equals(Object o) {
     @Override
     public int hashCode() {
       return Objects.hash(
-          queryText, column, boost, fuzziness, maxExpansions, operator, prefixLength);
+          queryText,
+          column,
+          boost,
+          fuzziness,
+          maxExpansions,
+          operator,
+          prefixLength,
+          documentGranularity);
     }
 
     @Override
@@ -206,6 +280,7 @@ public String toString() {
           .add("maxExpansions", maxExpansions)
           .add("operator", operator)
           .add("prefixLength", prefixLength)
+          .add("documentGranularity", documentGranularity)
           .toString();
     }
   }
@@ -215,8 +290,13 @@ public static final class PhraseQuery extends FullTextQuery {
     private final String queryText;
     private final String column;
     private final int slop;
+    private final Optional<DocumentGranularity> documentGranularity;
 
-    PhraseQuery(String queryText, String column, int slop) {
+    PhraseQuery(
+        String queryText,
+        String column,
+        int slop,
+        Optional<DocumentGranularity> documentGranularity) {
       Preconditions.checkArgument(
           queryText != null && !queryText.isEmpty(), "queryText must not be null or empty");
       Preconditions.checkArgument(
@@ -226,6 +306,7 @@ public static final class PhraseQuery extends FullTextQuery {
       this.queryText = queryText;
       this.column = column;
       this.slop = slop;
+      this.documentGranularity = Objects.requireNonNull(documentGranularity);
     }
 
     @Override
@@ -245,19 +326,25 @@ public int getSlop() {
       return slop;
     }
 
+    /** Returns the explicit granularity, or empty when query planning should infer it. */
+    public Optional<DocumentGranularity> getDocumentGranularity() {
+      return documentGranularity;
+    }
+
     @Override
     public boolean equals(Object o) {
       if (this == o) return true;
       if (!(o instanceof PhraseQuery)) return false;
       PhraseQuery other = (PhraseQuery) o;
       return slop == other.slop
+          && Objects.equals(documentGranularity, other.documentGranularity)
           && Objects.equals(queryText, other.queryText)
           && Objects.equals(column, other.column);
     }
 
     @Override
     public int hashCode() {
-      return Objects.hash(queryText, column, slop);
+      return Objects.hash(queryText, column, slop, documentGranularity);
     }
 
     @Override
@@ -267,6 +354,7 @@ public String toString() {
           .add("queryText", queryText)
           .add("column", column)
           .add("slop", slop)
+          .add("documentGranularity", documentGranularity)
           .toString();
     }
   }