more things

Signed-off-by: Adam Gutglick <adam@spiraldb.com>

Author: AdamGS

encodings/parquet-variant/src/array.rs

@@ -28,13 +28,29 @@ use vortex_mask::Mask;
 /// `ParquetVariantArray` preserves semi-structured data stored as Parquet Variant values in a
 /// lossless form and supports both unshredded and shredded layouts. Its storage matches the
 /// canonical extension type contract:
-/// - `metadata` is always present and non-nullable
+/// - `metadata` is always present and non-nullable.
 /// - `value` stores unshredded variant bytes when present
 /// - `typed_value` stores shredded data when present
 ///
 /// At least one of `value` or `typed_value` must be present. `typed_value` may be a primitive,
 /// list, or struct, with nested shredded children following the same recursive rules as the
 /// Arrow canonical extension type docs.
+///
+/// # Nullability
+///
+/// There are three independent levels of nullability in this encoding:
+///
+/// 1. **Outer validity** (`validity`): controls which *rows* of the variant column are null.
+///    This drives the `DType::Variant(Nullable | NonNullable)` of the enclosing `VariantArray`.
+///
+/// 2. **Value child nullability**: the `value` child is `Binary` and may be nullable or
+///    non-nullable. In partially-shredded layouts some rows have their data in `typed_value`
+///    instead, so the corresponding `value` slot is null — making the child nullable. This
+///    nullability is persisted in [`ParquetVariantMetadata::value_nullable`] so that
+///    serialization round-trips preserve the original Arrow field nullability.
+///
+/// 3. **Typed-value child nullability**: the `typed_value` child carries its own `DType`
+///    (which includes nullability).
 #[derive(Clone, Debug)]
 pub struct ParquetVariantArray {
     pub(crate) dtype: DType,

encodings/parquet-variant/src/vtable.rs

@@ -49,6 +49,12 @@ impl ParquetVariant {
 pub struct ParquetVariantMetadata {
     /// Whether the un-shredded `value` child is present.
     pub has_value: bool,
+    /// Whether the `value` child is nullable.
+    ///
+    /// In partially-shredded layouts, rows whose data lives entirely in `typed_value` have a
+    /// null `value` slot, so the Arrow field is marked nullable. This flag preserves that
+    /// distinction across serialization round-trips.
+    pub value_nullable: bool,
     /// DType of the shredded `typed_value`, if present.
     ///
     /// This is required to deserialize non-variant shredded children.
@@ -63,6 +69,9 @@ struct ParquetVariantMetadataProto {
     /// DType of the shredded `typed_value`, if present.
     #[prost(message, optional, tag = "2")]
     pub typed_value_dtype: Option<pb::DType>,
+    /// Whether the `value` child is nullable.
+    #[prost(bool, tag = "3")]
+    pub value_nullable: bool,
 }
 
 vtable!(ParquetVariant);
@@ -194,6 +203,10 @@ impl VTable for ParquetVariant {
     fn metadata(array: &ParquetVariantArray) -> VortexResult<Self::Metadata> {
         Ok(ParquetVariantMetadata {
             has_value: array.value.is_some(),
+            value_nullable: array
+                .value
+                .as_ref()
+                .is_some_and(|v| v.dtype().is_nullable()),
             typed_value_dtype: array.typed_value.as_ref().map(|tv| tv.dtype().clone()),
         })
     }
@@ -208,6 +221,7 @@ impl VTable for ParquetVariant {
             ParquetVariantMetadataProto {
                 has_value: metadata.has_value,
                 typed_value_dtype,
+                value_nullable: metadata.value_nullable,
             }
             .encode_to_vec(),
         ))
@@ -227,6 +241,7 @@ impl VTable for ParquetVariant {
         };
         Ok(ParquetVariantMetadata {
             has_value: proto.has_value,
+            value_nullable: proto.value_nullable,
             typed_value_dtype,
         })
     }
@@ -264,7 +279,11 @@ impl VTable for ParquetVariant {
         child_idx += 1;
 
         let value = if metadata.has_value {
-            let v = children.get(child_idx, &DType::Binary(Nullability::NonNullable), len)?;
+            let v = children.get(
+                child_idx,
+                &DType::Binary(metadata.value_nullable.into()),
+                len,
+            )?;
             child_idx += 1;
             Some(v)
         } else {

vortex-array/public-api.lock

@@ -22198,6 +22198,8 @@ pub vortex_array::CanonicalView::Struct(&'a vortex_array::arrays::StructArray)
 
 pub vortex_array::CanonicalView::VarBinView(&'a vortex_array::arrays::VarBinViewArray)
 
+pub vortex_array::CanonicalView::Variant(&'a vortex_array::arrays::variant::VariantArray)
+
 impl core::convert::AsRef<dyn vortex_array::DynArray> for vortex_array::CanonicalView<'_>
 
 pub fn vortex_array::CanonicalView<'_>::as_ref(&self) -> &dyn vortex_array::DynArray

vortex-array/src/canonical.rs

@@ -35,6 +35,7 @@ use crate::arrays::Struct;
 use crate::arrays::StructArray;
 use crate::arrays::VarBinView;
 use crate::arrays::VarBinViewArray;
+use crate::arrays::Variant;
 use crate::arrays::VariantArray;
 use crate::arrays::bool::BoolArrayParts;
 use crate::arrays::decimal::DecimalArrayParts;
@@ -950,6 +951,7 @@ pub enum CanonicalView<'a> {
     FixedSizeList(&'a FixedSizeListArray),
     Struct(&'a StructArray),
     Extension(&'a ExtensionArray),
+    Variant(&'a VariantArray),
 }
 
 impl From<CanonicalView<'_>> for Canonical {
@@ -964,6 +966,7 @@ impl From<CanonicalView<'_>> for Canonical {
             CanonicalView::FixedSizeList(a) => Canonical::FixedSizeList(a.clone()),
             CanonicalView::Struct(a) => Canonical::Struct(a.clone()),
             CanonicalView::Extension(a) => Canonical::Extension(a.clone()),
+            CanonicalView::Variant(a) => Canonical::Variant(a.clone()),
         }
     }
 }
@@ -980,6 +983,7 @@ impl AsRef<dyn DynArray> for CanonicalView<'_> {
             CanonicalView::FixedSizeList(a) => a.as_ref(),
             CanonicalView::Struct(a) => a.as_ref(),
             CanonicalView::Extension(a) => a.as_ref(),
+            CanonicalView::Variant(a) => a.as_ref(),
         }
     }
 }
@@ -999,6 +1003,7 @@ impl Matcher for AnyCanonical {
             || array.is::<FixedSizeList>()
             || array.is::<VarBinView>()
             || array.is::<Extension>()
+            || array.is::<Variant>()
     }
 
     fn try_match<'a>(array: &'a dyn DynArray) -> Option<Self::Match<'a>> {
@@ -1018,6 +1023,8 @@ impl Matcher for AnyCanonical {
             Some(CanonicalView::FixedSizeList(a))
         } else if let Some(a) = array.as_opt::<VarBinView>() {
             Some(CanonicalView::VarBinView(a))
+        } else if let Some(a) = array.as_opt::<Variant>() {
+            Some(CanonicalView::Variant(a))
         } else {
             array.as_opt::<Extension>().map(CanonicalView::Extension)
         }

vortex-array/src/scalar_fn/fns/cast/mod.rs

@@ -214,6 +214,9 @@ fn cast_canonical(
         CanonicalView::FixedSizeList(a) => <FixedSizeList as CastReduce>::cast(a, dtype),
         CanonicalView::Struct(a) => <Struct as CastKernel>::cast(a, dtype, ctx),
         CanonicalView::Extension(a) => <Extension as CastReduce>::cast(a, dtype),
+        CanonicalView::Variant(_) => {
+            vortex_bail!("Variant arrays don't support casting")
+        }
     }
 }