Handle any list builder kind in append_to_builder

`builder_with_capacity` produces a `ListViewBuilder` for `DType::List`, but
`List::append_to_builder` downcast the builder to `ListBuilder<u64>` and bailed
otherwise — so executing a `List`-encoded array into the canonical builder (the
one `builder_with_capacity` returns) failed. `ListView::append_to_builder` had
the mirror-image assumption, only accepting a `ListViewBuilder<u64, u64>`.

Add two macros that enumerate the possible list builder offset/size integer types
and try downcasting to each, rather than assuming the types produced by
`builder_with_capacity`:

- `match_each_list_builder!` enumerates `ListBuilder<O>`.
- `match_each_listview_builder!` enumerates `ListViewBuilder<O, S>`.

`List::append_to_builder` tries `ListBuilder` (its non-canonical, directly-supplied
builder) and otherwise canonicalizes and dispatches, routing the canonical
`ListViewBuilder` case through `ListView::append_to_builder`, which tries
`ListViewBuilder`. Either encoding works with any unsigned offset/size types.
Other encodings that yield list dtypes (constant, dict, chunked, ...) already
route here through the canonical-dispatch fallback.

Signed-off-by: Robert <robert@spiraldb.com>
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Mkrgj5SuJpaKBXK9jydtSc

Author: claude

vortex-array/src/arrays/list/vtable/mod.rs

@@ -16,6 +16,7 @@ use vortex_session::registry::CachedId;
 use crate::ArrayEq;
 use crate::ArrayHash;
 use crate::ArrayRef;
+use crate::Canonical;
 use crate::EqMode;
 use crate::ExecutionCtx;
 use crate::ExecutionResult;
@@ -35,7 +36,7 @@ use crate::arrays::list::compute::rules::PARENT_RULES;
 use crate::arrays::listview::list_view_from_list;
 use crate::buffer::BufferHandle;
 use crate::builders::ArrayBuilder;
-use crate::builders::ListBuilder;
+use crate::builders::match_each_list_builder;
 use crate::dtype::DType;
 use crate::dtype::Nullability;
 use crate::dtype::PType;
@@ -198,10 +199,14 @@ impl VTable for List {
         builder: &mut dyn ArrayBuilder,
         ctx: &mut ExecutionCtx,
     ) -> VortexResult<()> {
-        let Some(builder) = builder.as_any_mut().downcast_mut::<ListBuilder<u64>>() else {
-            vortex_bail!("append_to_builder for List requires a ListBuilder<u64, u64>");
-        };
-        builder.append_list_array(&array.into_owned().into_array(), ctx)
+        let array = array.into_owned().into_array();
+        // A directly-supplied `ListBuilder` (not the canonical builder for `DType::List`).
+        match_each_list_builder!(&array, builder, ctx);
+        // The canonical builder is `ListViewBuilder`; canonicalize and dispatch to `ListView`.
+        array
+            .execute::<Canonical>(ctx)?
+            .into_array()
+            .append_to_builder(builder, ctx)
     }
 }
 

vortex-array/src/arrays/listview/vtable/mod.rs

@@ -20,6 +20,7 @@ use crate::ArrayRef;
 use crate::EqMode;
 use crate::ExecutionCtx;
 use crate::ExecutionResult;
+use crate::IntoArray;
 use crate::array::Array;
 use crate::array::ArrayId;
 use crate::array::ArrayView;
@@ -34,7 +35,7 @@ use crate::arrays::listview::array::SLOT_NAMES;
 use crate::arrays::listview::compute::rules::PARENT_RULES;
 use crate::buffer::BufferHandle;
 use crate::builders::ArrayBuilder;
-use crate::builders::ListViewBuilder;
+use crate::builders::match_each_listview_builder;
 use crate::dtype::DType;
 use crate::dtype::Nullability;
 use crate::dtype::PType;
@@ -222,14 +223,12 @@ impl VTable for ListView {
         builder: &mut dyn ArrayBuilder,
         ctx: &mut ExecutionCtx,
     ) -> VortexResult<()> {
-        // `builder_with_capacity` always produces a `ListViewBuilder<u64, u64>` for `DType::List`.
-        let Some(builder) = builder
-            .as_any_mut()
-            .downcast_mut::<ListViewBuilder<u64, u64>>()
-        else {
-            vortex_bail!("append_to_builder for ListView requires a ListViewBuilder<u64, u64>");
-        };
-        builder.append_listview_array(&array.into_owned(), ctx)
+        let array = array.into_owned().into_array();
+        match_each_listview_builder!(&array, builder, ctx);
+        vortex_bail!(
+            "append_to_builder for ListView requires a ListViewBuilder, got a builder for {}",
+            builder.dtype()
+        )
     }
 
     fn reduce_parent(

vortex-array/src/builders/list.rs

@@ -173,8 +173,9 @@ impl<O: IntegerPType> ListBuilder<O> {
     /// [`ListViewArray`] and converting into the `ListArray` (`n + 1` offsets) layout.
     ///
     /// [`ListBuilder`] is not the canonical builder for [`DType::List`] (that is
-    /// [`ListViewBuilder`](crate::builders::ListViewBuilder)), so no encoding dispatches into it via
-    /// `append_to_builder`; this helper exists for direct callers such as tests.
+    /// [`ListViewBuilder`](crate::builders::ListViewBuilder)), so it is never produced by
+    /// [`builder_with_capacity`]. List encodings still dispatch into it via `match_each_list_builder`
+    /// when a caller supplies one directly.
     pub(crate) fn append_list_array(
         &mut self,
         array: &ArrayRef,
@@ -334,6 +335,7 @@ mod tests {
     use Nullability::Nullable;
     use vortex_buffer::buffer;
     use vortex_error::VortexExpect;
+    use vortex_error::VortexResult;
 
     use crate::IntoArray;
     use crate::array_session;
@@ -344,6 +346,8 @@ mod tests {
     use crate::arrays::listview::ListViewArrayExt;
     use crate::assert_arrays_eq;
     use crate::builders::ArrayBuilder;
+    use crate::builders::ListViewBuilder;
+    use crate::builders::builder_with_capacity;
     use crate::builders::list::ListArray;
     use crate::builders::list::ListBuilder;
     use crate::dtype::DType;
@@ -514,6 +518,50 @@ mod tests {
         );
     }
 
+    /// `append_to_builder` must handle any list builder kind without assuming the offset/size
+    /// integer types produced by `builder_with_capacity`. It appends a `List`-encoded array and a
+    /// `ListView`-encoded array into `ListViewBuilder`s and `ListBuilder`s with assorted (and
+    /// non-`u64`) offset/size types.
+    #[test]
+    fn test_append_to_builder_any_list_builder() -> VortexResult<()> {
+        let mut ctx = array_session().create_execution_ctx();
+
+        let list = ListArray::from_iter_opt_slow::<u64, _, _>(
+            [Some(vec![0, 1, 2]), None, Some(vec![4, 5])],
+            Arc::new(I32.into()),
+        )?
+        .into_array();
+        let listview = list
+            .clone()
+            .execute::<ListViewArray>(&mut ctx)?
+            .into_array();
+        let elem_dtype = || Arc::new(I32.into());
+
+        // `builder_with_capacity` produces a `ListViewBuilder` for `DType::List`; appending the
+        // `List`-encoded array must dispatch into it instead of bailing.
+        let mut listview_builder = builder_with_capacity(list.dtype(), list.len());
+        list.append_to_builder(listview_builder.as_mut(), &mut ctx)?;
+        assert_arrays_eq!(listview_builder.finish(), list, &mut ctx);
+
+        // Offset/size types are enumerated, not assumed: a `ListViewBuilder` with non-`u64`
+        // offset and size types must work for both source encodings.
+        let mut lv_u32_u8 = ListViewBuilder::<u32, u8>::with_capacity(elem_dtype(), Nullable, 8, 4);
+        list.append_to_builder(&mut lv_u32_u8, &mut ctx)?;
+        assert_arrays_eq!(lv_u32_u8.finish(), list, &mut ctx);
+
+        let mut lv_u16_u16 =
+            ListViewBuilder::<u16, u16>::with_capacity(elem_dtype(), Nullable, 8, 4);
+        listview.append_to_builder(&mut lv_u16_u16, &mut ctx)?;
+        assert_arrays_eq!(lv_u16_u16.finish(), list, &mut ctx);
+
+        // A `List`-encoded array appended into a `ListBuilder` with a non-`u64` offset type.
+        let mut list_builder = ListBuilder::<u32>::with_capacity(elem_dtype(), Nullable, 8, 4);
+        list.append_to_builder(&mut list_builder, &mut ctx)?;
+        assert_arrays_eq!(list_builder.finish(), list, &mut ctx);
+
+        Ok(())
+    }
+
     #[test]
     fn test_extend_builder() {
         test_extend_builder_gen::<i8>();

vortex-array/src/builders/mod.rs

@@ -284,3 +284,60 @@ pub fn builder_with_capacity_in(
     let _allocator = allocator;
     builder_with_capacity(dtype, capacity)
 }
+
+/// Append a list-typed `$array` into `$builder` if it is a [`ListBuilder`], enumerating the possible
+/// offset integer types and trying each downcast. On a match it `return`s the append result from the
+/// enclosing function; otherwise it falls through.
+///
+/// [`ListBuilder`] is never produced by [`builder_with_capacity`] (the canonical list builder is
+/// [`ListViewBuilder`]), so its offset type cannot be assumed — hence the enumeration. List offsets
+/// are always unsigned integers.
+macro_rules! match_each_list_builder {
+    ($array:expr, $builder:expr, $ctx:expr) => {{
+        $crate::builders::match_each_list_builder!(@types $array, $builder, $ctx, [u8, u16, u32, u64]);
+    }};
+    (@types $array:expr, $builder:expr, $ctx:expr, [$($O:ty),*]) => {{
+        $(
+            if let Some(builder) = $builder
+                .as_any_mut()
+                .downcast_mut::<$crate::builders::ListBuilder<$O>>()
+            {
+                return builder.append_list_array($array, $ctx);
+            }
+        )*
+    }};
+}
+pub(crate) use match_each_list_builder;
+
+/// Append a list-typed `$array` into `$builder` if it is a [`ListViewBuilder`], enumerating the
+/// possible offset and size integer types and trying each downcast. On a match it `return`s the
+/// append result from the enclosing function; otherwise it falls through.
+///
+/// [`builder_with_capacity`] produces a `ListViewBuilder<u64, u64>` for [`DType::List`], but a
+/// caller may supply different offset/size types, so the concrete type cannot be assumed — hence the
+/// enumeration. List offsets and sizes are always unsigned integers.
+macro_rules! match_each_listview_builder {
+    ($array:expr, $builder:expr, $ctx:expr) => {{
+        $crate::builders::match_each_listview_builder!(
+            @offsets $array, $builder, $ctx, [u8, u16, u32, u64], [u8, u16, u32, u64]
+        );
+    }};
+    // For each offset type `O`, expand over every size type `S` (cartesian product).
+    (@offsets $array:expr, $builder:expr, $ctx:expr, [$($O:ty),*], $sizes:tt) => {{
+        $(
+            $crate::builders::match_each_listview_builder!(@sizes $array, $builder, $ctx, $O, $sizes);
+        )*
+    }};
+    (@sizes $array:expr, $builder:expr, $ctx:expr, $O:ty, [$($S:ty),*]) => {{
+        $(
+            if let Some(builder) = $builder
+                .as_any_mut()
+                .downcast_mut::<$crate::builders::ListViewBuilder<$O, $S>>()
+            {
+                let listview = $array.clone().execute::<$crate::arrays::ListViewArray>($ctx)?;
+                return builder.append_listview_array(&listview, $ctx);
+            }
+        )*
+    }};
+}
+pub(crate) use match_each_listview_builder;