Fix StructArray / StructData after migration (#7305)

Fixes a decompression regression on wide struct datasets by removing
StructData metadata duplication and deriving struct metadata from the
outer cached dtype/len.

---------

Signed-off-by: Nicholas Gates <nick@nickgates.com>

Author: gatesn

Cargo.lock

@@ -100,7 +100,7 @@ pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult
 
             StructArray::try_new_with_dtype(
                 filtered_children,
-                struct_array.struct_fields(),
+                struct_array.struct_fields().clone(),
                 filter.iter().filter(|b| **b).map(|b| *b as usize).sum(),
                 validity,
             )

fuzz/src/array/filter.rs

@@ -122,7 +122,7 @@ pub fn mask_canonical_array(canonical: Canonical, mask: &Mask) -> VortexResult<A
             let new_validity = mask_validity(&array.validity()?, mask);
             StructArray::try_new_with_dtype(
                 array.unmasked_fields(),
-                array.struct_fields(),
+                array.struct_fields().clone(),
                 array.len(),
                 new_validity,
             )

fuzz/src/array/mask.rs

@@ -69,7 +69,7 @@ pub fn slice_canonical_array(
                 .collect::<VortexResult<Vec<_>>>()?;
             StructArray::try_new_with_dtype(
                 sliced_children,
-                struct_array.struct_fields(),
+                struct_array.struct_fields().clone(),
                 stop - start,
                 validity,
             )

fuzz/src/array/slice.rs

@@ -4170,7 +4170,7 @@ pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'
 
 pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String
 
-pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option<vortex_array::ArrayRef>]) -> vortex_error::VortexResult<()>
+pub fn vortex_array::arrays::Struct::validate(&self, _data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option<vortex_array::ArrayRef>]) -> vortex_error::VortexResult<()>
 
 impl vortex_array::ValidityVTable<vortex_array::arrays::Struct> for vortex_array::arrays::Struct
 
@@ -4198,22 +4198,14 @@ pub fn vortex_array::arrays::Struct::zip(if_true: vortex_array::ArrayView<'_, vo
 
 pub struct vortex_array::arrays::struct_::StructData
 
-impl vortex_array::arrays::struct_::StructData
-
-pub fn vortex_array::arrays::struct_::StructData::build(names: vortex_array::dtype::FieldNames, fields: impl core::convert::Into<alloc::sync::Arc<[vortex_array::ArrayRef]>>, length: usize, validity: vortex_array::validity::Validity) -> Self
-
-pub fn vortex_array::arrays::struct_::StructData::names(&self) -> &vortex_array::dtype::FieldNames
-
-pub fn vortex_array::arrays::struct_::StructData::new_fieldless_with_len(len: usize) -> Self
-
-pub unsafe fn vortex_array::arrays::struct_::StructData::new_unchecked(dtype: vortex_array::dtype::StructFields, fieldless_len: core::option::Option<usize>) -> Self
-
-pub fn vortex_array::arrays::struct_::StructData::validate(fields: &[vortex_array::ArrayRef], dtype: &vortex_array::dtype::StructFields, length: usize, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()>
-
 impl core::clone::Clone for vortex_array::arrays::struct_::StructData
 
 pub fn vortex_array::arrays::struct_::StructData::clone(&self) -> vortex_array::arrays::struct_::StructData
 
+impl core::default::Default for vortex_array::arrays::struct_::StructData
+
+pub fn vortex_array::arrays::struct_::StructData::default() -> vortex_array::arrays::struct_::StructData
+
 impl core::fmt::Debug for vortex_array::arrays::struct_::StructData
 
 pub fn vortex_array::arrays::struct_::StructData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
@@ -4242,7 +4234,7 @@ pub fn vortex_array::arrays::struct_::StructArrayExt::names(&self) -> &vortex_ar
 
 pub fn vortex_array::arrays::struct_::StructArrayExt::nullability(&self) -> vortex_array::dtype::Nullability
 
-pub fn vortex_array::arrays::struct_::StructArrayExt::struct_fields(&self) -> vortex_array::dtype::StructFields
+pub fn vortex_array::arrays::struct_::StructArrayExt::struct_fields(&self) -> &vortex_array::dtype::StructFields
 
 pub fn vortex_array::arrays::struct_::StructArrayExt::struct_validity(&self) -> vortex_array::validity::Validity
 
@@ -4262,7 +4254,7 @@ pub fn T::names(&self) -> &vortex_array::dtype::FieldNames
 
 pub fn T::nullability(&self) -> vortex_array::dtype::Nullability
 
-pub fn T::struct_fields(&self) -> vortex_array::dtype::StructFields
+pub fn T::struct_fields(&self) -> &vortex_array::dtype::StructFields
 
 pub fn T::struct_validity(&self) -> vortex_array::validity::Validity
 
@@ -6416,7 +6408,7 @@ pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'
 
 pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String
 
-pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option<vortex_array::ArrayRef>]) -> vortex_error::VortexResult<()>
+pub fn vortex_array::arrays::Struct::validate(&self, _data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option<vortex_array::ArrayRef>]) -> vortex_error::VortexResult<()>
 
 impl vortex_array::ValidityVTable<vortex_array::arrays::Struct> for vortex_array::arrays::Struct
 
@@ -19458,7 +19450,7 @@ pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'
 
 pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String
 
-pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option<vortex_array::ArrayRef>]) -> vortex_error::VortexResult<()>
+pub fn vortex_array::arrays::Struct::validate(&self, _data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option<vortex_array::ArrayRef>]) -> vortex_error::VortexResult<()>
 
 impl vortex_array::VTable for vortex_array::arrays::VarBin
 
@@ -20474,7 +20466,7 @@ pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'
 
 pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String
 
-pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option<vortex_array::ArrayRef>]) -> vortex_error::VortexResult<()>
+pub fn vortex_array::arrays::Struct::validate(&self, _data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option<vortex_array::ArrayRef>]) -> vortex_error::VortexResult<()>
 
 impl vortex_array::VTable for vortex_array::arrays::VarBin
 
@@ -23196,7 +23188,7 @@ pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'
 
 pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String
 
-pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option<vortex_array::ArrayRef>]) -> vortex_error::VortexResult<()>
+pub fn vortex_array::arrays::Struct::validate(&self, _data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option<vortex_array::ArrayRef>]) -> vortex_error::VortexResult<()>
 
 impl vortex_array::VTable for vortex_array::arrays::VarBin
 
@@ -24460,7 +24452,7 @@ pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'
 
 pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String
 
-pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option<vortex_array::ArrayRef>]) -> vortex_error::VortexResult<()>
+pub fn vortex_array::arrays::Struct::validate(&self, _data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option<vortex_array::ArrayRef>]) -> vortex_error::VortexResult<()>
 
 impl vortex_array::VTable for vortex_array::arrays::VarBin
 

vortex-array/public-api.lock

@@ -35,8 +35,13 @@ pub fn filter_struct(array: &StructArray, mask: &Arc<MaskValues>) -> StructArray
         .map(|a| a.len())
         .unwrap_or_else(|| mask.true_count());
 
-    StructArray::try_new_with_dtype(fields, array.struct_fields(), length, filtered_validity)
-        .vortex_expect("filtered StructArray fields have consistent lengths")
+    StructArray::try_new_with_dtype(
+        fields,
+        array.struct_fields().clone(),
+        length,
+        filtered_validity,
+    )
+    .vortex_expect("filtered StructArray fields have consistent lengths")
 }
 
 #[cfg(test)]

vortex-array/src/arrays/filter/execute/struct_.rs

@@ -185,7 +185,7 @@ fn mask_validity_struct(
     let fields = array.unmasked_fields();
     let struct_fields = array.struct_fields();
     // SAFETY: We're only changing validity, not the data structure
-    Ok(unsafe { StructArray::new_unchecked(fields, struct_fields, len, new_validity) })
+    Ok(unsafe { StructArray::new_unchecked(fields, struct_fields.clone(), len, new_validity) })
 }
 
 fn mask_validity_extension(