perf(encoding): auto-select sparse structural pages

docs/src/format/file/encoding.md

@@ -355,15 +355,18 @@ semantics after normalization.
 
 #### Writer Selection
 
-Writers may emit this layout only for Lance 2.3+ fields that explicitly set
-`lance-encoding:structural-encoding=sparse`. The same request is an input error for earlier file versions. This metadata
-controls writer selection only: readers always use `PageLayout` to determine the layout of an encoded page and must
-not use field metadata for that decision.
-
-Sparse selection is opt-in. The default Lance 2.3 writer policy remains unchanged, as do explicit `miniblock` and
-`fullzip` requests. Writers normalize Arrow validity and list structure once, then use that semantic structure for
-either dense repetition/definition serialization or sparse position/count serialization. They should choose the
-validity polarity with the lower semantic encoded cost; ties use null positions.
+Writers may emit this layout only for Lance 2.3+ fields. A field can request it explicitly with
+`lance-encoding:structural-encoding=sparse`; the same request is an input error for earlier file versions. Without an
+explicit structural encoding, the Lance 2.3 writer selects sparse only when the dense mini-block repetition/definition
+budget would split the page or one top-level row exceeds that budget, and only when the value path is supported by the
+sparse writer. Explicit `miniblock`, `fullzip`, and `sparse` requests are not changed by this automatic policy. Lance
+2.2 and earlier writers never select sparse.
+
+Unsupported sparse value paths, including dictionary values and variable-width packed structs, retain their dense
+behavior. Writers normalize Arrow validity and list structure once. Within-budget dense pages do not build sparse
+position/count plans. When sparse is selected, writers choose the validity polarity with the lower semantic encoded
+cost; ties use null positions. Field metadata controls writer selection only: readers always use `PageLayout` to
+determine the layout of an encoded page and must not use field metadata for that decision.
 
 Pages without a value payload keep the existing canonical `ConstantLayout`: structural-only types such as an empty
 struct, and leaf pages whose visible values are all null, do not emit `SparseLayout`. An explicitly sparse page with
@@ -643,7 +646,7 @@ options. However, they can also be set in the field metadata in the schema.
 | `lance-encoding:dict-values-compression-level` | Integers (scheme dependent) | Varies by scheme | Compression level for dictionary values general compression                             |
 | `lance-encoding:general`             | `off`, `on`                          | `off`            | Whether to apply general compression.                                                   |
 | `lance-encoding:packed`              | Any string                           | Not set          | Whether to apply packed struct encoding (see above).                                    |
-| `lance-encoding:structural-encoding` | `miniblock`, `fullzip`               | Not set          | Force a particular structural encoding to be applied (only useful for testing purposes) |
+| `lance-encoding:structural-encoding` | `miniblock`, `fullzip`, `sparse`     | Not set          | Force a particular structural encoding to be applied (only useful for testing purposes) |
 
 ### Configuration Details
 

rust/lance-encoding/src/encodings/logical/primitive.rs

@@ -92,6 +92,7 @@ pub mod blob;
 pub mod constant;
 pub mod dict;
 pub mod fullzip;
+mod layout;
 pub mod miniblock;
 pub(crate) mod sparse;
 
@@ -3813,7 +3814,10 @@ enum PrimitivePageStructure {
         repdef: SerializedRepDefs,
         unsplittable_miniblock_levels: Option<u64>,
     },
-    Sparse(sparse::SparseStructuralPlan),
+    Sparse {
+        plan: sparse::SparseStructuralPlan,
+        prepared_values: Option<sparse::writer::PreparedSparseValues>,
+    },
 }
 
 // A primitive page after optional structural splitting.
@@ -5407,7 +5411,10 @@ impl PrimitiveStructuralEncoder {
                 repdef,
                 unsplittable_miniblock_levels,
             } => (repdef, unsplittable_miniblock_levels),
-            PrimitivePageStructure::Sparse(plan) => {
+            PrimitivePageStructure::Sparse {
+                plan,
+                prepared_values,
+            } => {
                 log::debug!(
                     "Encoding column {} with {} visible items ({} rows) using sparse layout",
                     column_idx,
@@ -5418,7 +5425,14 @@ impl PrimitiveStructuralEncoder {
                     column_idx,
                     &field,
                     compression_strategy.as_ref(),
-                    DataBlock::from_arrays(&arrays, num_values),
+                    prepared_values.map_or_else(
+                        || {
+                            sparse::writer::SparseValueInput::Unprepared(DataBlock::from_arrays(
+                                &arrays, num_values,
+                            ))
+                        },
+                        sparse::writer::SparseValueInput::Prepared,
+                    ),
                     plan,
                     row_number,
                     num_rows,
@@ -5734,35 +5748,78 @@ impl PrimitiveStructuralEncoder {
         let is_simple_validity = repdefs.iter().all(|rd| rd.is_simple_validity());
         let has_repdef_info = repdefs.iter().any(|rd| !rd.is_empty());
         let normalized = RepDefBuilder::normalize(repdefs);
-        let requests_sparse = self
-            .encoding_metadata
-            .get(STRUCTURAL_ENCODING_META_KEY)
+        let requested_encoding = self.encoding_metadata.get(STRUCTURAL_ENCODING_META_KEY);
+        let requests_sparse = requested_encoding
             .is_some_and(|requested| requested.eq_ignore_ascii_case(STRUCTURAL_ENCODING_SPARSE));
         let sparse_plan = requests_sparse
             .then(|| sparse::writer::plan(&normalized, num_values))
             .transpose()?;
-        let pages = match sparse_plan {
-            Some(plan) if !sparse::writer::uses_constant_layout(&plan, &self.field) => {
-                vec![PrimitivePageData {
+        let pages = if let Some(plan) = sparse_plan
+            && !sparse::writer::uses_constant_layout(&plan, &self.field)
+        {
+            vec![PrimitivePageData {
+                arrays,
+                structure: PrimitivePageStructure::Sparse {
+                    plan,
+                    prepared_values: None,
+                },
+                row_number,
+                num_rows,
+            }]
+        } else {
+            let (repdef, structural_plan) = normalized.serialize_with_structural_plan(
+                miniblock::max_repdef_levels_per_chunk,
+                num_rows,
+                num_values,
+            )?;
+            let automatic_sparse = layout::select_automatic_sparse(
+                self.version.resolve(),
+                requested_encoding.map(String::as_str),
+                &structural_plan,
+                || {
+                    let data = DataBlock::from_arrays(&arrays, num_values);
+                    if !sparse::writer::supports_value_block(&data) {
+                        return Ok(None);
+                    }
+                    let prepared_values = match sparse::writer::prepare_values(
+                        &self.field,
+                        self.compression_strategy.as_ref(),
+                        data,
+                        self.support_large_chunk,
+                    ) {
+                        Ok(prepared_values) => prepared_values,
+                        Err(error) => {
+                            trace!(
+                                "Keeping column {} on its dense structural path because sparse value preparation is unavailable: {}",
+                                self.column_index, error
+                            );
+                            return Ok(None);
+                        }
+                    };
+                    let plan = sparse::writer::plan(&normalized, num_values)?;
+                    if sparse::writer::uses_constant_layout(&plan, &self.field) {
+                        return Ok(None);
+                    }
+                    Ok(Some((plan, prepared_values)))
+                },
+            )?;
+            match automatic_sparse {
+                Some((plan, prepared_values)) => vec![PrimitivePageData {
                     arrays,
-                    structure: PrimitivePageStructure::Sparse(plan),
+                    structure: PrimitivePageStructure::Sparse {
+                        plan,
+                        prepared_values: Some(prepared_values),
+                    },
                     row_number,
                     num_rows,
-                }]
-            }
-            _ => {
-                let (repdef, structural_plan) = normalized.serialize_with_structural_plan(
-                    miniblock::max_repdef_levels_per_chunk,
-                    num_rows,
-                    num_values,
-                )?;
-                Self::split_structural_pages_for_miniblock_budget(
+                }],
+                None => Self::split_structural_pages_for_miniblock_budget(
                     arrays,
                     repdef,
                     structural_plan,
                     row_number,
                     num_rows,
-                )?
+                )?,
             }
         };
 

rust/lance-encoding/src/encodings/logical/primitive/layout.rs

@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use lance_core::Result;
+
+use crate::{repdef::StructuralPagePlan, version::LanceFileVersion};
+
+/// Runs automatic sparse planning only after the dense mini-block budget makes it useful.
+pub(super) fn select_automatic_sparse<T>(
+    version: LanceFileVersion,
+    requested_encoding: Option<&str>,
+    dense_plan: &StructuralPagePlan,
+    candidate: impl FnOnce() -> Result<Option<T>>,
+) -> Result<Option<T>> {
+    if version < LanceFileVersion::V2_3
+        || requested_encoding.is_some()
+        || !matches!(
+            dense_plan,
+            StructuralPagePlan::Split(_) | StructuralPagePlan::UnsplittableOverBudget(_)
+        )
+    {
+        return Ok(None);
+    }
+    candidate()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::constants::{
+        STRUCTURAL_ENCODING_FULLZIP, STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_SPARSE,
+    };
+
+    #[test]
+    fn within_budget_does_not_construct_sparse_candidate() {
+        let selected = select_automatic_sparse::<()>(
+            LanceFileVersion::V2_3,
+            None,
+            &StructuralPagePlan::Fits,
+            || panic!("within-budget pages must not construct sparse candidates"),
+        )
+        .unwrap();
+        assert!(selected.is_none());
+    }
+
+    #[test]
+    fn over_budget_selects_only_eligible_candidates() {
+        let split = StructuralPagePlan::Split(Vec::new());
+        let selected =
+            select_automatic_sparse(LanceFileVersion::V2_3, None, &split, || Ok(Some(42))).unwrap();
+        assert_eq!(selected, Some(42));
+
+        let ineligible =
+            select_automatic_sparse::<()>(LanceFileVersion::V2_3, None, &split, || Ok(None))
+                .unwrap();
+        assert!(ineligible.is_none());
+
+        let unsplittable = StructuralPagePlan::UnsplittableOverBudget(70_000);
+        let selected =
+            select_automatic_sparse(LanceFileVersion::V2_3, None, &unsplittable, || Ok(Some(7)))
+                .unwrap();
+        assert_eq!(selected, Some(7));
+    }
+
+    #[test]
+    fn explicit_modes_and_lance_2_2_do_not_auto_select() {
+        let split = StructuralPagePlan::Split(Vec::new());
+        for requested in [
+            STRUCTURAL_ENCODING_MINIBLOCK,
+            STRUCTURAL_ENCODING_FULLZIP,
+            STRUCTURAL_ENCODING_SPARSE,
+        ] {
+            let selected = select_automatic_sparse::<()>(
+                LanceFileVersion::V2_3,
+                Some(requested),
+                &split,
+                || panic!("explicit modes must not invoke automatic sparse planning"),
+            )
+            .unwrap();
+            assert!(selected.is_none());
+        }
+
+        let selected = select_automatic_sparse::<()>(LanceFileVersion::V2_2, None, &split, || {
+            panic!("Lance 2.2 must not invoke automatic sparse planning")
+        })
+        .unwrap();
+        assert!(selected.is_none());
+    }
+}