Variant DType (#6912)

This PR includes initial support for the Variant DType, as described in
the [Variant
RFC](https://github.com/vortex-data/rfcs/blob/develop/accepted/0015-variant-type.md).
It includes most of the required boilerplate and initial structure for
this new dtype.

It includes the following changes:
1. New dtype
2. serialization for the dtype
3. Scalar variant and new scalar value
4. A lot of code paths that aren't supported yet and error accordingly.

---------

Signed-off-by: Adam Gutglick <adam@spiraldb.com>

Author: AdamGS

encodings/sparse/src/canonical.rs

@@ -50,6 +50,7 @@ use vortex_buffer::buffer_mut;
 use vortex_error::VortexError;
 use vortex_error::VortexExpect;
 use vortex_error::VortexResult;
+use vortex_error::vortex_bail;
 use vortex_error::vortex_panic;
 
 use crate::ConstantArray;
@@ -117,6 +118,7 @@ pub(super) fn execute_sparse(
             execute_sparse_fixed_size_list(array, *nullability, ctx)?
         }
         DType::Extension(_ext_dtype) => todo!(),
+        DType::Variant(_) => vortex_bail!("Sparse canonicalization does not support Variant"),
     })
 }
 

fuzz/src/array/compare.rs

@@ -141,7 +141,7 @@ pub fn compare_canonical_array(
             )
             .into_array()
         }
-        d @ (DType::Null | DType::Extension(_)) => {
+        d @ (DType::Null | DType::Extension(_) | DType::Variant(_)) => {
             unreachable!("DType {d} not supported for fuzzing")
         }
     }

fuzz/src/array/filter.rs

@@ -115,7 +115,7 @@ pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult
             }
             take_canonical_array_non_nullable_indices(array, indices.as_slice())
         }
-        d @ (DType::Null | DType::Extension(_)) => {
+        d @ (DType::Null | DType::Extension(_) | DType::Variant(_)) => {
             unreachable!("DType {d} not supported for fuzzing")
         }
     }

fuzz/src/array/mod.rs

@@ -508,6 +508,8 @@ fn actions_for_dtype(dtype: &DType) -> HashSet<ActionType> {
             // Extension types delegate to storage dtype, support most operations
             ActionType::iter().collect()
         }
+        // Currently, no support at all
+        DType::Variant(_) => unreachable!("Variant dtype shouldn't be fuzzed"),
     }
 }
 

fuzz/src/array/search_sorted.rs

@@ -131,7 +131,7 @@ pub fn search_sorted_canonical_array(
                 .collect::<VortexResult<Vec<_>>>()?;
             scalar_vals.search_sorted(&scalar.cast(array.dtype())?, side)
         }
-        d @ (DType::Null | DType::Extension(_)) => {
+        d @ (DType::Null | DType::Extension(_) | DType::Variant(_)) => {
             unreachable!("DType {d} not supported for fuzzing")
         }
     }

fuzz/src/array/slice.rs

@@ -114,7 +114,7 @@ pub fn slice_canonical_array(
                 .into_array(),
             )
         }
-        d @ (DType::Null | DType::Extension(_)) => {
+        d @ (DType::Null | DType::Extension(_) | DType::Variant(_)) => {
             unreachable!("DType {d} not supported for fuzzing")
         }
     }

fuzz/src/array/sort.rs

@@ -81,7 +81,7 @@ pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult<ArrayRef> {
             });
             take_canonical_array_non_nullable_indices(array, &sort_indices)
         }
-        d @ (DType::Null | DType::Extension(_)) => {
+        d @ (DType::Null | DType::Extension(_) | DType::Variant(_)) => {
             unreachable!("DType {d} not supported for fuzzing")
         }
     }

fuzz/src/array/take.rs

@@ -138,7 +138,7 @@ pub fn take_canonical_array(array: &ArrayRef, indices: &[Option<usize>]) -> Vort
             }
             Ok(builder.finish())
         }
-        d @ (DType::Null | DType::Extension(_)) => {
+        d @ (DType::Null | DType::Extension(_) | DType::Variant(_)) => {
             unreachable!("DType {d} not supported for fuzzing")
         }
     }

vortex-array/public-api.lock

@@ -10374,6 +10374,8 @@ pub vortex_array::dtype::DType::Struct(vortex_array::dtype::StructFields, vortex
 
 pub vortex_array::dtype::DType::Utf8(vortex_array::dtype::Nullability)
 
+pub vortex_array::dtype::DType::Variant(vortex_array::dtype::Nullability)
+
 impl vortex_array::dtype::DType
 
 pub const vortex_array::dtype::DType::BYTES: Self
@@ -10450,6 +10452,8 @@ pub fn vortex_array::dtype::DType::is_unsigned_int(&self) -> bool
 
 pub fn vortex_array::dtype::DType::is_utf8(&self) -> bool
 
+pub fn vortex_array::dtype::DType::is_variant(&self) -> bool
+
 pub fn vortex_array::dtype::DType::list(dtype: impl core::convert::Into<vortex_array::dtype::DType>, nullability: vortex_array::dtype::Nullability) -> Self
 
 pub fn vortex_array::dtype::DType::nullability(&self) -> vortex_array::dtype::Nullability
@@ -15354,6 +15358,8 @@ pub vortex_array::scalar::ScalarValue::Primitive(vortex_array::scalar::PValue)
 
 pub vortex_array::scalar::ScalarValue::Utf8(vortex_buffer::string::BufferString)
 
+pub vortex_array::scalar::ScalarValue::Variant(alloc::boxed::Box<vortex_array::scalar::Scalar>)
+
 impl vortex_array::scalar::ScalarValue
 
 pub fn vortex_array::scalar::ScalarValue::as_binary(&self) -> &vortex_buffer::ByteBuffer
@@ -15368,6 +15374,8 @@ pub fn vortex_array::scalar::ScalarValue::as_primitive(&self) -> &vortex_array::
 
 pub fn vortex_array::scalar::ScalarValue::as_utf8(&self) -> &vortex_buffer::string::BufferString
 
+pub fn vortex_array::scalar::ScalarValue::as_variant(&self) -> &vortex_array::scalar::Scalar
+
 pub fn vortex_array::scalar::ScalarValue::into_binary(self) -> vortex_buffer::ByteBuffer
 
 pub fn vortex_array::scalar::ScalarValue::into_bool(self) -> bool
@@ -15380,6 +15388,8 @@ pub fn vortex_array::scalar::ScalarValue::into_primitive(self) -> vortex_array::
 
 pub fn vortex_array::scalar::ScalarValue::into_utf8(self) -> vortex_buffer::string::BufferString
 
+pub fn vortex_array::scalar::ScalarValue::into_variant(self) -> vortex_array::scalar::Scalar
+
 impl vortex_array::scalar::ScalarValue
 
 pub fn vortex_array::scalar::ScalarValue::from_proto(value: &vortex_proto::scalar::ScalarValue, dtype: &vortex_array::dtype::DType, session: &vortex_session::VortexSession) -> vortex_error::VortexResult<core::option::Option<Self>>
@@ -16054,6 +16064,10 @@ pub fn vortex_array::scalar::Scalar::as_utf8(&self) -> vortex_array::scalar::Utf
 
 pub fn vortex_array::scalar::Scalar::as_utf8_opt(&self) -> core::option::Option<vortex_array::scalar::Utf8Scalar<'_>>
 
+pub fn vortex_array::scalar::Scalar::as_variant(&self) -> vortex_array::scalar::VariantScalar<'_>
+
+pub fn vortex_array::scalar::Scalar::as_variant_opt(&self) -> core::option::Option<vortex_array::scalar::VariantScalar<'_>>
+
 impl vortex_array::scalar::Scalar
 
 pub fn vortex_array::scalar::Scalar::binary(buffer: impl core::convert::Into<vortex_buffer::ByteBuffer>, nullability: vortex_array::dtype::Nullability) -> Self
@@ -16080,6 +16094,8 @@ pub fn vortex_array::scalar::Scalar::try_utf8<B>(str: B, nullability: vortex_arr
 
 pub fn vortex_array::scalar::Scalar::utf8<B>(str: B, nullability: vortex_array::dtype::Nullability) -> Self where B: core::convert::Into<vortex_buffer::string::BufferString>
 
+pub fn vortex_array::scalar::Scalar::variant(value: vortex_array::scalar::Scalar) -> Self
+
 impl vortex_array::scalar::Scalar
 
 pub fn vortex_array::scalar::Scalar::cast(&self, target_dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult<vortex_array::scalar::Scalar>
@@ -16744,6 +16760,34 @@ impl<'a> core::hash::Hash for vortex_array::scalar::Utf8Scalar<'a>
 
 pub fn vortex_array::scalar::Utf8Scalar<'a>::hash<__H: core::hash::Hasher>(&self, state: &mut __H)
 
+pub struct vortex_array::scalar::VariantScalar<'a>
+
+impl<'a> vortex_array::scalar::VariantScalar<'a>
+
+pub fn vortex_array::scalar::VariantScalar<'a>::dtype(&self) -> &'a vortex_array::dtype::DType
+
+pub fn vortex_array::scalar::VariantScalar<'a>::is_null(&self) -> bool
+
+pub fn vortex_array::scalar::VariantScalar<'a>::is_variant_null(&self) -> core::option::Option<bool>
+
+pub fn vortex_array::scalar::VariantScalar<'a>::is_zero(&self) -> core::option::Option<bool>
+
+pub fn vortex_array::scalar::VariantScalar<'a>::value(&self) -> core::option::Option<&'a vortex_array::scalar::Scalar>
+
+impl core::fmt::Display for vortex_array::scalar::VariantScalar<'_>
+
+pub fn vortex_array::scalar::VariantScalar<'_>::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
+
+impl<'a> core::clone::Clone for vortex_array::scalar::VariantScalar<'a>
+
+pub fn vortex_array::scalar::VariantScalar<'a>::clone(&self) -> vortex_array::scalar::VariantScalar<'a>
+
+impl<'a> core::fmt::Debug for vortex_array::scalar::VariantScalar<'a>
+
+pub fn vortex_array::scalar::VariantScalar<'a>::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
+
+impl<'a> core::marker::Copy for vortex_array::scalar::VariantScalar<'a>
+
 pub trait vortex_array::scalar::ScalarTruncation: core::marker::Send + core::marker::Sized
 
 pub fn vortex_array::scalar::ScalarTruncation::from_scalar(value: vortex_array::scalar::Scalar) -> vortex_error::VortexResult<core::option::Option<Self>>

vortex-array/src/arrays/arbitrary.rs

@@ -162,7 +162,10 @@ fn random_array_chunk(
             random_fixed_size_list(u, elem_dtype, *list_size, *null, chunk_len)
         }
         DType::Extension(..) => {
-            todo!("Extension arrays are not implemented")
+            unimplemented!("Extension arrays are not implemented")
+        }
+        DType::Variant(_) => {
+            unimplemented!("Variant arrays are not implemented")
         }
     }
 }