From f2da41001d64bae558dae9dd2d289d8ded1f0392 Mon Sep 17 00:00:00 2001 From: cakeni <2150015994@qq.com> Date: Wed, 12 Aug 2026 00:00:39 +0800 Subject: [PATCH] fix(variant): support fixed-size list shredding --- parquet-variant-compute/src/variant_array.rs | 22 ++++++++++++++++---- parquet-variant-compute/src/variant_get.rs | 17 ++++++++++++--- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/parquet-variant-compute/src/variant_array.rs b/parquet-variant-compute/src/variant_array.rs index 7af504572e5b..33c0989a0f8d 100644 --- a/parquet-variant-compute/src/variant_array.rs +++ b/parquet-variant-compute/src/variant_array.rs @@ -291,7 +291,7 @@ impl VariantArray { /// binary_view /// /// 3. An optional field named `typed_value` which can be any primitive type - /// or be a list, large_list, list_view or struct + /// or be a list, large_list, fixed_size_list, list_view or struct /// /// NOTE: It is also permissible for the metadata field to be /// Dictionary-Encoded, preferably (but not required) with an index type of @@ -1236,7 +1236,7 @@ fn canonicalize_and_verify_data_type(data_type: &DataType) -> Result borrow!(), - FixedSizeBinary(_) | FixedSizeList(..) => fail!(), + FixedSizeBinary(_) => fail!(), // List-like containers and struct are allowed, maps and unions are not List(field) => match canonicalize_and_verify_field(field)? { @@ -1255,6 +1255,10 @@ fn canonicalize_and_verify_data_type(data_type: &DataType) -> Result borrow!(), Cow::Owned(new_field) => Cow::Owned(DataType::LargeListView(new_field)), }, + FixedSizeList(field, size) => match canonicalize_and_verify_field(field)? { + Cow::Borrowed(_) => borrow!(), + Cow::Owned(new_field) => Cow::Owned(DataType::FixedSizeList(new_field, *size)), + }, // Struct is used by the internal layout, and can also represent a shredded variant object. Struct(fields) => { // Avoid allocation unless at least one field changes, to avoid unnecessary deep cloning @@ -1334,8 +1338,8 @@ mod test { use super::*; use arrow::array::{ BinaryArray, BinaryViewArray, Decimal32Array, Decimal64Array, Decimal128Array, - FixedSizeBinaryArray, Int32Array, Int64Array, LargeBinaryArray, LargeListArray, - LargeListViewArray, ListArray, ListViewArray, Time64MicrosecondArray, + FixedSizeBinaryArray, FixedSizeListArray, Int32Array, Int64Array, LargeBinaryArray, + LargeListArray, LargeListViewArray, ListArray, ListViewArray, Time64MicrosecondArray, }; use arrow::buffer::{OffsetBuffer, ScalarBuffer}; use arrow_schema::{Field, Fields}; @@ -1535,6 +1539,10 @@ mod test { DataType::LargeList(make_item_binary_view()), DataType::ListView(make_item_binary_view()), DataType::LargeListView(make_item_binary_view()), + // Fixed-size list items + DataType::FixedSizeList(make_item_binary(), 2), + DataType::FixedSizeList(make_large_binary(), 2), + DataType::FixedSizeList(make_item_binary_view(), 2), ]; for input in cases { @@ -1577,6 +1585,12 @@ mod test { values, None, )) as ArrayRef, + Arc::new(FixedSizeListArray::new( + Arc::new(Field::new("item", DataType::Int64, true)), + 2, + Arc::new(Int64Array::from(vec![Some(1), None, Some(3), Some(4)])), + None, + )) as ArrayRef, ]; for typed_value in typed_values { diff --git a/parquet-variant-compute/src/variant_get.rs b/parquet-variant-compute/src/variant_get.rs index d062d0ff0816..e0feee209750 100644 --- a/parquet-variant-compute/src/variant_get.rs +++ b/parquet-variant-compute/src/variant_get.rs @@ -16,8 +16,8 @@ // under the License. use arrow::{ array::{ - self, Array, ArrayRef, GenericListArray, GenericListViewArray, ListLikeArray, StructArray, - UInt64Array, make_array, + self, Array, ArrayRef, FixedSizeListArray, GenericListArray, GenericListViewArray, + ListLikeArray, StructArray, UInt64Array, make_array, }, buffer::NullBuffer, compute::{CastOptions, take}, @@ -171,6 +171,9 @@ pub(crate) fn follow_shredded_path_element( DataType::LargeListView(_) => take_list_like_index_as_shredding_state::< GenericListViewArray, >(typed_value.as_ref(), *index)?, + DataType::FixedSizeList(_, _) => take_list_like_index_as_shredding_state::< + FixedSizeListArray, + >(typed_value.as_ref(), *index)?, _ => { // JSONPath semantics: indexing a non-list yields no match. return Ok(missing_path_step()); @@ -1937,12 +1940,13 @@ mod test { type ShreddedListLikeArrayGen = fn() -> ArrayRef; type ShreddedListLikeCase = (&'static str, ShreddedListLikeArrayGen); - fn shredded_list_like_cases() -> [ShreddedListLikeCase; 4] { + fn shredded_list_like_cases() -> [ShreddedListLikeCase; 5] { [ ("list", shredded_list_variant_array), ("large_list", shredded_large_list_variant_array), ("list_view", shredded_list_view_variant_array), ("large_list_view", shredded_large_list_view_variant_array), + ("fixed_size_list", shredded_fixed_size_list_variant_array), ] } @@ -2108,6 +2112,13 @@ mod test { )))) } + fn shredded_fixed_size_list_variant_array() -> ArrayRef { + shredded_list_like_variant_array(DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Utf8, true)), + 2, + )) + } + fn shredded_struct_with_list_variant_array() -> ArrayRef { let json_rows: ArrayRef = Arc::new(StringArray::from(vec![ Some(r#"{"a": ["comedy", "drama"]}"#),