wip: chunked iter exe

Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>

Author: joseph-isaacs

vortex-array/public-api.lock

@@ -3364,7 +3364,7 @@ pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype:
 
 pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::Array) -> &vortex_array::dtype::DType
 
-pub fn vortex_array::arrays::patched::Patched::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
+pub fn vortex_array::arrays::patched::Patched::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
 
 pub fn vortex_array::arrays::patched::Patched::execute_parent(array: &vortex_array::vtable::Array<Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::ArrayRef>>
 
@@ -3416,6 +3416,10 @@ pub fn vortex_array::arrays::patched::PatchedArray::from_array_and_patches(inner
 
 impl vortex_array::arrays::patched::PatchedArray
 
+pub fn vortex_array::arrays::patched::PatchedArray::into_parts(self) -> vortex_array::arrays::patched::PatchedArrayParts
+
+impl vortex_array::arrays::patched::PatchedArray
+
 pub fn vortex_array::arrays::patched::PatchedArray::to_array(&self) -> vortex_array::ArrayRef
 
 impl core::clone::Clone for vortex_array::arrays::patched::PatchedArray
@@ -3444,6 +3448,22 @@ impl vortex_array::IntoArray for vortex_array::arrays::patched::PatchedArray
 
 pub fn vortex_array::arrays::patched::PatchedArray::into_array(self) -> vortex_array::ArrayRef
 
+pub struct vortex_array::arrays::patched::PatchedArrayParts
+
+pub vortex_array::arrays::patched::PatchedArrayParts::indices: vortex_array::ArrayRef
+
+pub vortex_array::arrays::patched::PatchedArrayParts::inner: vortex_array::ArrayRef
+
+pub vortex_array::arrays::patched::PatchedArrayParts::lane_offsets: vortex_array::ArrayRef
+
+pub vortex_array::arrays::patched::PatchedArrayParts::len: usize
+
+pub vortex_array::arrays::patched::PatchedArrayParts::n_lanes: usize
+
+pub vortex_array::arrays::patched::PatchedArrayParts::offset: usize
+
+pub vortex_array::arrays::patched::PatchedArrayParts::values: vortex_array::ArrayRef
+
 pub struct vortex_array::arrays::patched::PatchedMetadata
 
 impl core::clone::Clone for vortex_array::arrays::patched::PatchedMetadata
@@ -4148,7 +4168,7 @@ pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &v
 
 pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceArray) -> &vortex_array::dtype::DType
 
-pub fn vortex_array::arrays::slice::Slice::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
+pub fn vortex_array::arrays::slice::Slice::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
 
 pub fn vortex_array::arrays::slice::Slice::execute_parent(array: &vortex_array::vtable::Array<Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::ArrayRef>>
 
@@ -7332,7 +7352,7 @@ pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype:
 
 pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::Array) -> &vortex_array::dtype::DType
 
-pub fn vortex_array::arrays::patched::Patched::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
+pub fn vortex_array::arrays::patched::Patched::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
 
 pub fn vortex_array::arrays::patched::Patched::execute_parent(array: &vortex_array::vtable::Array<Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::ArrayRef>>
 
@@ -7384,6 +7404,10 @@ pub fn vortex_array::arrays::patched::PatchedArray::from_array_and_patches(inner
 
 impl vortex_array::arrays::patched::PatchedArray
 
+pub fn vortex_array::arrays::patched::PatchedArray::into_parts(self) -> vortex_array::arrays::patched::PatchedArrayParts
+
+impl vortex_array::arrays::patched::PatchedArray
+
 pub fn vortex_array::arrays::patched::PatchedArray::to_array(&self) -> vortex_array::ArrayRef
 
 impl core::clone::Clone for vortex_array::arrays::patched::PatchedArray
@@ -7942,7 +7966,7 @@ pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &v
 
 pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceArray) -> &vortex_array::dtype::DType
 
-pub fn vortex_array::arrays::slice::Slice::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
+pub fn vortex_array::arrays::slice::Slice::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
 
 pub fn vortex_array::arrays::slice::Slice::execute_parent(array: &vortex_array::vtable::Array<Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::ArrayRef>>
 
@@ -22324,7 +22348,7 @@ pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype:
 
 pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::Array) -> &vortex_array::dtype::DType
 
-pub fn vortex_array::arrays::patched::Patched::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
+pub fn vortex_array::arrays::patched::Patched::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
 
 pub fn vortex_array::arrays::patched::Patched::execute_parent(array: &vortex_array::vtable::Array<Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::ArrayRef>>
 
@@ -22444,7 +22468,7 @@ pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &v
 
 pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceArray) -> &vortex_array::dtype::DType
 
-pub fn vortex_array::arrays::slice::Slice::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
+pub fn vortex_array::arrays::slice::Slice::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
 
 pub fn vortex_array::arrays::slice::Slice::execute_parent(array: &vortex_array::vtable::Array<Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::ArrayRef>>
 
@@ -23768,7 +23792,7 @@ pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype:
 
 pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::Array) -> &vortex_array::dtype::DType
 
-pub fn vortex_array::arrays::patched::Patched::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
+pub fn vortex_array::arrays::patched::Patched::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
 
 pub fn vortex_array::arrays::patched::Patched::execute_parent(array: &vortex_array::vtable::Array<Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::ArrayRef>>
 
@@ -23888,7 +23912,7 @@ pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &v
 
 pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceArray) -> &vortex_array::dtype::DType
 
-pub fn vortex_array::arrays::slice::Slice::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
+pub fn vortex_array::arrays::slice::Slice::execute(array: alloc::sync::Arc<vortex_array::vtable::Array<Self>>, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ExecutionResult>
 
 pub fn vortex_array::arrays::slice::Slice::execute_parent(array: &vortex_array::vtable::Array<Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::ArrayRef>>
 

vortex-array/src/arrays/filter/vtable.rs

@@ -13,12 +13,14 @@ use vortex_error::vortex_panic;
 use vortex_mask::Mask;
 use vortex_session::VortexSession;
 
+use crate::AnyCanonical;
 use crate::ArrayEq;
 use crate::ArrayHash;
 use crate::ArrayRef;
 use crate::DynArray;
 use crate::IntoArray;
 use crate::Precision;
+use crate::arrays::filter::array::CHILD_SLOT;
 use crate::arrays::filter::array::FilterArray;
 use crate::arrays::filter::array::NUM_SLOTS;
 use crate::arrays::filter::array::SLOT_NAMES;
@@ -30,6 +32,7 @@ use crate::buffer::BufferHandle;
 use crate::dtype::DType;
 use crate::executor::ExecutionCtx;
 use crate::executor::ExecutionResult;
+use crate::require_child;
 use crate::scalar::Scalar;
 use crate::serde::ArrayChildren;
 use crate::stats::StatsSetRef;
@@ -151,14 +154,18 @@ impl VTable for Filter {
         if let Some(canonical) = execute_filter_fast_paths(&array, ctx)? {
             return Ok(ExecutionResult::done(canonical));
         }
+
+        let array = require_child!(array, array.child(), CHILD_SLOT => AnyCanonical);
+
         let Mask::Values(mask_values) = &array.mask else {
             unreachable!("`execute_filter_fast_paths` handles AllTrue and AllFalse")
         };
 
-        // We rely on the optimization pass that runs prior to this execution for filter pushdown,
-        // so now we can just execute the filter without worrying.
+        // Child is pre-canonicalized — apply the filter directly.
+        debug_assert!(array.child().is_canonical());
+        let child = array.child().to_canonical()?;
         Ok(ExecutionResult::done(
-            execute_filter(array.child().clone().execute(ctx)?, mask_values).into_array(),
+            execute_filter(child, mask_values).into_array(),
         ))
     }
 

vortex-array/src/arrays/list/vtable/mod.rs

@@ -18,7 +18,9 @@ use crate::IntoArray;
 use crate::Precision;
 use crate::ProstMetadata;
 use crate::arrays::ListArray;
+use crate::arrays::Primitive;
 use crate::arrays::list::array::NUM_SLOTS;
+use crate::arrays::list::array::OFFSETS_SLOT;
 use crate::arrays::list::array::SLOT_NAMES;
 use crate::arrays::list::array::VALIDITY_SLOT;
 use crate::arrays::list::compute::PARENT_KERNELS;
@@ -32,6 +34,7 @@ use crate::hash::ArrayEq;
 use crate::hash::ArrayHash;
 use crate::metadata::DeserializeMetadata;
 use crate::metadata::SerializeMetadata;
+use crate::require_child;
 use crate::serde::ArrayChildren;
 use crate::stats::StatsSetRef;
 use crate::validity::Validity;
@@ -193,6 +196,7 @@ impl VTable for List {
     }
 
     fn execute(array: Arc<Array<Self>>, ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult> {
+        let array = require_child!(array, array.offsets(), OFFSETS_SLOT => Primitive);
         Ok(ExecutionResult::done(
             list_view_from_list(ListArray::clone(&array), ctx)?.into_array(),
         ))

vortex-array/src/arrays/masked/vtable/mod.rs

@@ -13,13 +13,14 @@ use vortex_error::vortex_ensure;
 use vortex_error::vortex_panic;
 use vortex_session::VortexSession;
 
+use crate::AnyCanonical;
 use crate::ArrayRef;
-use crate::Canonical;
 use crate::EmptyMetadata;
 use crate::IntoArray;
 use crate::Precision;
 use crate::arrays::ConstantArray;
 use crate::arrays::MaskedArray;
+use crate::arrays::masked::array::CHILD_SLOT;
 use crate::arrays::masked::array::NUM_SLOTS;
 use crate::arrays::masked::array::SLOT_NAMES;
 use crate::arrays::masked::compute::rules::PARENT_RULES;
@@ -30,6 +31,7 @@ use crate::executor::ExecutionCtx;
 use crate::executor::ExecutionResult;
 use crate::hash::ArrayEq;
 use crate::hash::ArrayHash;
+use crate::require_child;
 use crate::scalar::Scalar;
 use crate::serde::ArrayChildren;
 use crate::stats::StatsSetRef;
@@ -162,7 +164,10 @@ impl VTable for Masked {
         // While we could manually convert the dtype, `mask_validity_canonical` is already O(1) for
         // `AllTrue` masks (no data copying), so there's no benefit.
 
-        let child = array.child().clone().execute::<Canonical>(ctx)?;
+        let array = require_child!(array, array.child(), CHILD_SLOT => AnyCanonical);
+
+        debug_assert!(array.child().is_canonical());
+        let child = array.child().to_canonical()?;
         Ok(ExecutionResult::done(
             mask_validity_canonical(child, &validity_mask, ctx)?.into_array(),
         ))

vortex-array/src/arrays/patched/array.rs

@@ -205,6 +205,47 @@ impl PatchedArray {
     }
 }
 
+/// The owned parts of a [`PatchedArray`], produced by [`PatchedArray::into_parts`].
+pub struct PatchedArrayParts {
+    /// The inner array being patched.
+    pub inner: ArrayRef,
+    /// The lane offsets array (u32).
+    pub lane_offsets: ArrayRef,
+    /// The patch indices array (u16).
+    pub indices: ArrayRef,
+    /// The patch values array.
+    pub values: ArrayRef,
+    /// Number of lanes.
+    pub n_lanes: usize,
+    /// Offset into the first chunk.
+    pub offset: usize,
+    /// Logical length.
+    pub len: usize,
+}
+
+impl PatchedArray {
+    /// Consume this array into its owned parts.
+    pub fn into_parts(mut self) -> PatchedArrayParts {
+        PatchedArrayParts {
+            inner: self.slots[INNER_SLOT]
+                .take()
+                .vortex_expect("PatchedArray inner slot"),
+            lane_offsets: self.slots[LANE_OFFSETS_SLOT]
+                .take()
+                .vortex_expect("PatchedArray lane_offsets slot"),
+            indices: self.slots[INDICES_SLOT]
+                .take()
+                .vortex_expect("PatchedArray indices slot"),
+            values: self.slots[VALUES_SLOT]
+                .take()
+                .vortex_expect("PatchedArray values slot"),
+            n_lanes: self.n_lanes,
+            offset: self.offset,
+            len: self.len,
+        }
+    }
+}
+
 impl PatchedArray {
     /// Returns a reference to the base array being patched.
     #[inline]

vortex-array/src/arrays/patched/vtable/mod.rs

@@ -28,10 +28,16 @@ use crate::IntoArray;
 use crate::Precision;
 use crate::ProstMetadata;
 use crate::SerializeMetadata;
+use crate::arrays::Primitive;
 use crate::arrays::PrimitiveArray;
 use crate::arrays::patched::PatchedArray;
+use crate::arrays::patched::PatchedArrayParts;
+use crate::arrays::patched::array::INDICES_SLOT;
+use crate::arrays::patched::array::INNER_SLOT;
+use crate::arrays::patched::array::LANE_OFFSETS_SLOT;
 use crate::arrays::patched::array::NUM_SLOTS;
 use crate::arrays::patched::array::SLOT_NAMES;
+use crate::arrays::patched::array::VALUES_SLOT;
 use crate::arrays::patched::compute::rules::PARENT_RULES;
 use crate::arrays::patched::vtable::kernels::PARENT_KERNELS;
 use crate::arrays::primitive::PrimitiveArrayParts;
@@ -42,6 +48,7 @@ use crate::dtype::DType;
 use crate::dtype::NativePType;
 use crate::dtype::PType;
 use crate::match_each_native_ptype;
+use crate::require_child;
 use crate::serde::ArrayChildren;
 use crate::stats::ArrayStats;
 use crate::stats::StatsSetRef;
@@ -288,45 +295,53 @@ impl VTable for Patched {
         Ok(())
     }
 
-    fn execute(array: Arc<Array<Self>>, ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult> {
-        let inner = array
-            .base_array()
-            .clone()
-            .execute::<Canonical>(ctx)?
-            .into_primitive();
+    fn execute(array: Arc<Array<Self>>, _ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult> {
+        let array = require_child!(array, array.base_array(), INNER_SLOT => Primitive);
+        let array = require_child!(array, array.lane_offsets(), LANE_OFFSETS_SLOT => Primitive);
+        let array = require_child!(array, array.patch_indices(), INDICES_SLOT => Primitive);
+        let array = require_child!(array, array.patch_values(), VALUES_SLOT => Primitive);
+
+        let PatchedArrayParts {
+            inner,
+            lane_offsets,
+            indices,
+            values,
+            n_lanes,
+            offset,
+            len,
+        } = Arc::unwrap_or_clone(array).into_inner().into_parts();
+
+        let inner = inner
+            .try_into::<Primitive>()
+            .ok()
+            .vortex_expect("base_array pre-canonicalized to Primitive");
+        let lane_offsets = lane_offsets
+            .try_into::<Primitive>()
+            .ok()
+            .vortex_expect("lane_offsets pre-canonicalized to Primitive");
+        let indices = indices
+            .try_into::<Primitive>()
+            .ok()
+            .vortex_expect("patch_indices pre-canonicalized to Primitive");
+        let values = values
+            .try_into::<Primitive>()
+            .ok()
+            .vortex_expect("patch_values pre-canonicalized to Primitive");
 
         let PrimitiveArrayParts {
             buffer,
             ptype,
             validity,
         } = inner.into_parts();
 
-        let lane_offsets = array
-            .lane_offsets()
-            .clone()
-            .execute::<PrimitiveArray>(ctx)?;
-        let indices = array
-            .patch_indices()
-            .clone()
-            .execute::<PrimitiveArray>(ctx)?;
-
-        // TODO(aduffy): add support for non-primitive PatchedArray patches application (?)
-        let values = array
-            .patch_values()
-            .clone()
-            .execute::<PrimitiveArray>(ctx)?;
-
         let patched_values = match_each_native_ptype!(values.ptype(), |V| {
-            let offset = array.offset;
-            let len = array.len;
-
             let mut output = Buffer::<V>::from_byte_buffer(buffer.unwrap_host()).into_mut();
 
             apply_patches_primitive::<V>(
                 &mut output,
                 offset,
                 len,
-                array.n_lanes,
+                n_lanes,
                 lane_offsets.as_slice::<u32>(),
                 indices.as_slice::<u16>(),
                 values.as_slice::<V>(),