Remove metadata from DynArray (#7435)

connortsui20 · web-flow · commit f7472016ec08 · 2026-04-14T20:09:35.000Z
## Summary

Apparently we just don't need this now? And we want to use
`array_serialize` to get metadata instead.

This also fixes a few incorrect tests that we had.

## API Changes

Removes the `metadata` method on `DynArray`.

## Testing

Fixes some tests.

---------

Signed-off-by: Connor Tsui &lt;connor.tsui20@gmail.com&gt;
diff --git a/encodings/alp/src/alp/plugin.rs b/encodings/alp/src/alp/plugin.rs
@@ -133,7 +133,7 @@ mod tests {
 
         let array = alp_encoded.as_array();
 
-        let metadata = array.metadata(&SESSION)?.unwrap_or_default();
+        let metadata = SESSION.array_serialize(array)?.unwrap();
         let children = array.children();
         let buffers = array
             .buffers()
@@ -182,7 +182,7 @@ mod tests {
 
         let array = alp_encoded.as_array();
 
-        let metadata = array.metadata(&SESSION)?.unwrap_or_default();
+        let metadata = SESSION.array_serialize(array)?.unwrap();
         let children = array.children();
         let buffers = array
             .buffers()
@@ -213,7 +213,7 @@ mod tests {
     fn primitive_array_returns_error() {
         let array = PrimitiveArray::from_iter([1.0f64, 2.0, 3.0]).into_array();
 
-        let metadata = array.metadata(&SESSION).unwrap().unwrap_or_default();
+        let metadata = SESSION.array_serialize(&array).unwrap().unwrap();
         let children = array.children();
         let buffers = array
             .buffers()
diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs
@@ -352,7 +352,6 @@ impl From<Vec<Option<bool>>> for ByteBoolData {
 mod tests {
     use vortex_array::ArrayContext;
     use vortex_array::IntoArray;
-    use vortex_array::LEGACY_SESSION;
     use vortex_array::assert_arrays_eq;
     use vortex_array::serde::SerializeOptions;
     use vortex_array::serde::SerializedArray;
@@ -407,7 +406,7 @@ mod tests {
         let serialized = array
             .clone()
             .into_array()
-            .serialize(&ctx, &LEGACY_SESSION, &SerializeOptions::default())
+            .serialize(&ctx, &session, &SerializeOptions::default())
             .unwrap();
 
         let mut concat = ByteBufferMut::empty();
diff --git a/encodings/fastlanes/src/bitpacking/plugin.rs b/encodings/fastlanes/src/bitpacking/plugin.rs
@@ -133,7 +133,7 @@ mod tests {
 
         let array = bitpacked.as_array();
 
-        let metadata = array.metadata(&SESSION)?.unwrap_or_default();
+        let metadata = SESSION.array_serialize(array)?.unwrap();
         let children = array.children();
         let buffers = array
             .buffers()
@@ -182,7 +182,7 @@ mod tests {
 
         let array = bitpacked.as_array();
 
-        let metadata = array.metadata(&SESSION)?.unwrap_or_default();
+        let metadata = SESSION.array_serialize(array)?.unwrap();
         let children = array.children();
         let buffers = array
             .buffers()
@@ -212,7 +212,7 @@ mod tests {
     fn primitive_array_returns_error() -> VortexResult<()> {
         let array = PrimitiveArray::from_iter([1i32, 2, 3]).into_array();
 
-        let metadata = array.metadata(&SESSION)?.unwrap_or_default();
+        let metadata = SESSION.array_serialize(&array)?.unwrap();
         let children = array.children();
         let buffers = array
             .buffers()
diff --git a/encodings/parquet-variant/src/vtable.rs b/encodings/parquet-variant/src/vtable.rs
@@ -244,13 +244,13 @@ mod tests {
     use vortex_array::IntoArray;
     use vortex_array::Precision;
     use vortex_array::arrays::VarBinViewArray;
-    use vortex_array::arrays::Variant;
     use vortex_array::arrays::VariantArray;
     use vortex_array::dtype::DType;
     use vortex_array::dtype::Nullability;
     use vortex_array::dtype::PType;
     use vortex_array::serde::SerializeOptions;
     use vortex_array::serde::SerializedArray;
+    use vortex_array::session::ArraySession;
     use vortex_array::session::ArraySessionExt;
     use vortex_array::validity::Validity;
     use vortex_buffer::BitBuffer;
@@ -261,11 +261,14 @@ mod tests {
 
     use crate::ParquetVariant;
     use crate::array::ParquetVariantArrayExt;
+
     fn roundtrip(array: ArrayRef) -> ArrayRef {
         let dtype = array.dtype().clone();
         let len = array.len();
 
-        let session = VortexSession::empty().with::<vortex_array::session::ArraySession>();
+        let session = VortexSession::empty().with::<ArraySession>();
+        session.arrays().register(ParquetVariant);
+
         let ctx = ArrayContext::empty();
         let serialized = array
             .serialize(&ctx, &session, &SerializeOptions::default())
@@ -276,8 +279,6 @@ mod tests {
             concat.extend_from_slice(buf.as_ref());
         }
         let concat = concat.freeze();
-        session.arrays().register(ParquetVariant);
-        session.arrays().register(Variant);
 
         let parts = SerializedArray::try_from(concat).unwrap();
         parts
diff --git a/encodings/pco/src/lib.rs b/encodings/pco/src/lib.rs
@@ -5,8 +5,6 @@ mod array;
 mod compute;
 mod rules;
 mod slice;
-#[cfg(test)]
-mod test;
 
 pub use array::*;
 
@@ -35,3 +33,6 @@ pub struct PcoMetadata {
     #[prost(message, repeated, tag = "2")]
     pub chunks: Vec<PcoChunkInfo>,
 }
+
+#[cfg(test)]
+mod tests;
diff --git a/encodings/pco/src/tests.rs b/encodings/pco/src/tests.rs
@@ -184,7 +184,7 @@ fn test_serde() -> VortexResult<()> {
     let bytes = pco
         .serialize(
             &context,
-            &LEGACY_SESSION,
+            &SESSION,
             &SerializeOptions {
                 offset: 0,
                 include_padding: true,
diff --git a/encodings/runend/src/arrow.rs b/encodings/runend/src/arrow.rs
@@ -90,6 +90,7 @@ mod tests {
     use vortex_array::search_sorted::SearchSorted;
     use vortex_array::search_sorted::SearchSortedSide;
     use vortex_array::session::ArraySession;
+    use vortex_array::session::ArraySessionExt;
     use vortex_array::validity::Validity;
     use vortex_buffer::Buffer;
     use vortex_buffer::buffer;
@@ -99,8 +100,11 @@ mod tests {
     use crate::RunEnd;
     use crate::ops::find_slice_end_index;
 
-    static SESSION: LazyLock<VortexSession> =
-        LazyLock::new(|| VortexSession::empty().with::<ArraySession>());
+    static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
+        let session = VortexSession::empty().with::<ArraySession>();
+        session.arrays().register(RunEnd);
+        session
+    });
 
     fn decode_run_array<R: RunEndIndexType>(
         array: &RunArray<R>,
diff --git a/encodings/zstd/src/zstd_buffers.rs b/encodings/zstd/src/zstd_buffers.rs
@@ -59,9 +59,9 @@ impl ZstdBuffers {
         session: &VortexSession,
     ) -> VortexResult<ZstdBuffersArray> {
         let encoding_id = array.encoding_id();
-        let metadata = array
-            .metadata(session)?
-            .ok_or_else(|| vortex_err!("Array does not support serialization"))?;
+        let metadata = session
+            .array_serialize(array)?
+            .ok_or_else(|| vortex_err!("[ZstdBuffers]: Array does not support serialization"))?;
         let buffer_handles = array.buffer_handles();
         let children = array.children();
 
diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock
@@ -22024,8 +22024,6 @@ pub fn vortex_array::ArrayRef::is_valid(&self, index: usize) -> vortex_error::Vo
 
 pub fn vortex_array::ArrayRef::len(&self) -> usize
 
-pub fn vortex_array::ArrayRef::metadata(&self, session: &vortex_session::VortexSession) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
-
 pub fn vortex_array::ArrayRef::metadata_fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
 
 pub fn vortex_array::ArrayRef::named_buffers(&self) -> alloc::vec::Vec<(alloc::string::String, vortex_array::buffer::BufferHandle)>
diff --git a/vortex-array/src/array/erased.rs b/vortex-array/src/array/erased.rs
@@ -15,7 +15,6 @@ use vortex_error::vortex_ensure;
 use vortex_error::vortex_err;
 use vortex_error::vortex_panic;
 use vortex_mask::Mask;
-use vortex_session::VortexSession;
 
 use crate::AnyCanonical;
 use crate::Array;
@@ -561,11 +560,6 @@ impl ArrayRef {
         self.0.slot_name(self, idx)
     }
 
-    /// Returns the serialized metadata of the array.
-    pub fn metadata(&self, session: &VortexSession) -> VortexResult<Option<Vec<u8>>> {
-        self.0.metadata(self, session)
-    }
-
     /// Formats a human-readable metadata description.
     pub fn metadata_fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         self.0.metadata_fmt(f)
diff --git a/vortex-array/src/array/mod.rs b/vortex-array/src/array/mod.rs
@@ -13,7 +13,6 @@ use vortex_error::VortexResult;
 use vortex_error::vortex_ensure;
 use vortex_error::vortex_err;
 use vortex_error::vortex_panic;
-use vortex_session::VortexSession;
 use vortex_session::registry::Id;
 
 use crate::ExecutionCtx;
@@ -132,10 +131,6 @@ pub(crate) trait DynArray: 'static + private::Sealed + Send + Sync + Debug {
     /// Returns the name of the slot at the given index.
     fn slot_name(&self, this: &ArrayRef, idx: usize) -> String;
 
-    /// Returns the serialized metadata of the array, or `None` if the array does not
-    /// support serialization.
-    fn metadata(&self, this: &ArrayRef, session: &VortexSession) -> VortexResult<Option<Vec<u8>>>;
-
     /// Formats a human-readable metadata description.
     fn metadata_fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result;
 
@@ -341,11 +336,6 @@ impl<V: VTable> DynArray for ArrayInner<V> {
         V::slot_name(view, idx)
     }
 
-    fn metadata(&self, this: &ArrayRef, session: &VortexSession) -> VortexResult<Option<Vec<u8>>> {
-        let view = unsafe { ArrayView::new_unchecked(this, &self.data) };
-        V::serialize(view, session)
-    }
-
     fn metadata_fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         std::fmt::Display::fmt(&self.data, f)
     }
diff --git a/vortex-array/src/arrow/executor/run_end.rs b/vortex-array/src/arrow/executor/run_end.rs
@@ -24,6 +24,7 @@ use crate::IntoArray;
 use crate::arrays::Constant;
 use crate::arrays::ConstantArray;
 use crate::arrow::ArrowArrayExecutor;
+use crate::session::ArraySessionExt;
 
 /// The encoding ID used by `vortex-runend`. We match on this string to avoid a crate dependency.
 const VORTEX_RUNEND_ID: &str = "vortex.runend";
@@ -79,8 +80,9 @@ fn run_end_to_arrow(
     ctx: &mut ExecutionCtx,
 ) -> VortexResult<ArrowArrayRef> {
     let length = array.len();
-    let metadata_bytes = array
-        .metadata(ctx.session())?
+    let metadata_bytes = ctx
+        .session()
+        .array_serialize(&array)?
         .ok_or_else(|| vortex_err!("RunEndArray missing metadata"))?;
     let metadata = RunEndMetadata::decode(&*metadata_bytes)
         .map_err(|e| vortex_err!("Failed to decode RunEndMetadata: {e}"))?;
diff --git a/vortex-array/src/serde.rs b/vortex-array/src/serde.rs
@@ -170,16 +170,6 @@ impl<'a> ArrayNodeFlatBuffer<'a> {
         session: &'a VortexSession,
         array: &'a ArrayRef,
     ) -> VortexResult<Self> {
-        // Depth-first traversal of the array to ensure it supports serialization.
-        // FIXME(ngates): this serializes the metadata and throws it away!
-        for child in array.depth_first_traversal() {
-            if child.metadata(session)?.is_none() {
-                vortex_bail!(
-                    "Array {} does not support serialization",
-                    child.encoding_id()
-                );
-            }
-        }
         let n_buffers_recursive = array.nbuffers_recursive();
         if n_buffers_recursive > u16::MAX as usize {
             vortex_bail!(
@@ -210,13 +200,13 @@ impl<'a> ArrayNodeFlatBuffer<'a> {
                 )
             })?;
 
-        let metadata = self.array.metadata(self.session)?.ok_or_else(|| {
+        let metadata_bytes = self.session.array_serialize(self.array)?.ok_or_else(|| {
             vortex_err!(
                 "Array {} does not support serialization",
                 self.array.encoding_id()
             )
         })?;
-        let metadata = Some(fbb.create_vector(metadata.as_slice()));
+        let metadata = Some(fbb.create_vector(metadata_bytes.as_slice()));
 
         // Assign buffer indices for all child arrays.
         let nbuffers = u16::try_from(self.array.nbuffers())
@@ -701,101 +691,3 @@ impl TryFrom<BufferHandle> for SerializedArray {
         Self::try_from(value.try_to_host_sync()?)
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use std::sync::LazyLock;
-
-    use flatbuffers::FlatBufferBuilder;
-    use vortex_session::VortexSession;
-    use vortex_session::registry::ReadContext;
-
-    use super::SerializeOptions;
-    use super::SerializedArray;
-    use crate::ArrayContext;
-    use crate::array::ArrayId;
-    use crate::dtype::DType;
-    use crate::dtype::Nullability;
-    use crate::flatbuffers as fba;
-    use crate::session::ArraySession;
-
-    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
-
-    #[test]
-    fn unknown_array_encoding_allow_unknown() {
-        let mut fbb = FlatBufferBuilder::new();
-
-        let child_metadata = fbb.create_vector(&[9u8]);
-        let child = fba::ArrayNode::create(
-            &mut fbb,
-            &fba::ArrayNodeArgs {
-                encoding: 1,
-                metadata: Some(child_metadata),
-                children: None,
-                buffers: None,
-                stats: None,
-            },
-        );
-
-        let children = fbb.create_vector(&[child]);
-        let metadata = fbb.create_vector(&[1u8, 2, 3]);
-        let root = fba::ArrayNode::create(
-            &mut fbb,
-            &fba::ArrayNodeArgs {
-                encoding: 0,
-                metadata: Some(metadata),
-                children: Some(children),
-                buffers: None,
-                stats: None,
-            },
-        );
-        let array = fba::Array::create(
-            &mut fbb,
-            &fba::ArrayArgs {
-                root: Some(root),
-                buffers: None,
-            },
-        );
-        fbb.finish_minimal(array);
-        let (buf, start) = fbb.collapse();
-        let tree = vortex_buffer::ByteBuffer::from(buf).slice(start..);
-
-        let ser = SerializedArray::from_array_tree(tree).unwrap();
-        let ctx = ReadContext::new([
-            ArrayId::new("vortex.test.foreign_array"),
-            ArrayId::new("vortex.test.foreign_child"),
-        ]);
-        let session = VortexSession::empty()
-            .with::<ArraySession>()
-            .allow_unknown();
-
-        let decoded = ser
-            .decode(&DType::Variant(Nullability::Nullable), 5, &ctx, &session)
-            .unwrap();
-        assert_eq!(decoded.encoding_id().as_ref(), "vortex.test.foreign_array");
-        assert_eq!(decoded.nchildren(), 1);
-        assert_eq!(
-            decoded.nth_child(0).unwrap().encoding_id().as_ref(),
-            "vortex.test.foreign_child"
-        );
-        assert_eq!(decoded.metadata(&SESSION).unwrap().unwrap(), vec![1, 2, 3]);
-        assert_eq!(
-            decoded
-                .nth_child(0)
-                .unwrap()
-                .metadata(&SESSION)
-                .unwrap()
-                .unwrap(),
-            vec![9]
-        );
-
-        let serialized = decoded
-            .serialize(
-                &ArrayContext::default(),
-                &SESSION,
-                &SerializeOptions::default(),
-            )
-            .unwrap();
-        assert!(!serialized.is_empty());
-    }
-}
diff --git a/vortex-array/src/session/mod.rs b/vortex-array/src/session/mod.rs
diff --git a/vortex-cuda/src/hybrid_dispatch/mod.rs b/vortex-cuda/src/hybrid_dispatch/mod.rs
diff --git a/vortex-python/src/io.rs b/vortex-python/src/io.rs
