more things

Signed-off-by: Adam Gutglick <adam@spiraldb.com>

Author: AdamGS

encodings/parquet-variant/src/array.rs

@@ -28,13 +28,29 @@ use vortex_mask::Mask;
 /// `ParquetVariantArray` preserves semi-structured data stored as Parquet Variant values in a
 /// lossless form and supports both unshredded and shredded layouts. Its storage matches the
 /// canonical extension type contract:
-/// - `metadata` is always present and non-nullable
+/// - `metadata` is always present and non-nullable.
 /// - `value` stores unshredded variant bytes when present
 /// - `typed_value` stores shredded data when present
 ///
 /// At least one of `value` or `typed_value` must be present. `typed_value` may be a primitive,
 /// list, or struct, with nested shredded children following the same recursive rules as the
 /// Arrow canonical extension type docs.
+///
+/// # Nullability
+///
+/// There are three independent levels of nullability in this encoding:
+///
+/// 1. **Outer validity** (`validity`): controls which *rows* of the variant column are null.
+///    This drives the `DType::Variant(Nullable | NonNullable)` of the enclosing `VariantArray`.
+///
+/// 2. **Value child nullability**: the `value` child is `Binary` and may be nullable or
+///    non-nullable. In partially-shredded layouts some rows have their data in `typed_value`
+///    instead, so the corresponding `value` slot is null — making the child nullable. This
+///    nullability is persisted in [`ParquetVariantMetadata::value_nullable`] so that
+///    serialization round-trips preserve the original Arrow field nullability.
+///
+/// 3. **Typed-value child nullability**: the `typed_value` child carries its own `DType`
+///    (which includes nullability).
 #[derive(Clone, Debug)]
 pub struct ParquetVariantArray {
     pub(crate) dtype: DType,

encodings/parquet-variant/src/vtable.rs

@@ -49,6 +49,12 @@ impl ParquetVariant {
 pub struct ParquetVariantMetadata {
     /// Whether the un-shredded `value` child is present.
     pub has_value: bool,
+    /// Whether the `value` child is nullable.
+    ///
+    /// In partially-shredded layouts, rows whose data lives entirely in `typed_value` have a
+    /// null `value` slot, so the Arrow field is marked nullable. This flag preserves that
+    /// distinction across serialization round-trips.
+    pub value_nullable: bool,
     /// DType of the shredded `typed_value`, if present.
     ///
     /// This is required to deserialize non-variant shredded children.
@@ -63,6 +69,9 @@ struct ParquetVariantMetadataProto {
     /// DType of the shredded `typed_value`, if present.
     #[prost(message, optional, tag = "2")]
     pub typed_value_dtype: Option<pb::DType>,
+    /// Whether the `value` child is nullable.
+    #[prost(bool, tag = "3")]
+    pub value_nullable: bool,
 }
 
 vtable!(ParquetVariant);
@@ -194,6 +203,10 @@ impl VTable for ParquetVariant {
     fn metadata(array: &ParquetVariantArray) -> VortexResult<Self::Metadata> {
         Ok(ParquetVariantMetadata {
             has_value: array.value.is_some(),
+            value_nullable: array
+                .value
+                .as_ref()
+                .is_some_and(|v| v.dtype().is_nullable()),
             typed_value_dtype: array.typed_value.as_ref().map(|tv| tv.dtype().clone()),
         })
     }
@@ -208,6 +221,7 @@ impl VTable for ParquetVariant {
             ParquetVariantMetadataProto {
                 has_value: metadata.has_value,
                 typed_value_dtype,
+                value_nullable: metadata.value_nullable,
             }
             .encode_to_vec(),
         ))
@@ -227,6 +241,7 @@ impl VTable for ParquetVariant {
         };
         Ok(ParquetVariantMetadata {
             has_value: proto.has_value,
+            value_nullable: proto.value_nullable,
             typed_value_dtype,
         })
     }
@@ -264,7 +279,11 @@ impl VTable for ParquetVariant {
         child_idx += 1;
 
         let value = if metadata.has_value {
-            let v = children.get(child_idx, &DType::Binary(Nullability::NonNullable), len)?;
+            let v = children.get(
+                child_idx,
+                &DType::Binary(metadata.value_nullable.into()),
+                len,
+            )?;
             child_idx += 1;
             Some(v)
         } else {

vortex-array/src/canonical.rs

@@ -1002,6 +1002,7 @@ impl Matcher for AnyCanonical {
             || array.is::<VarBinView>()
             || array.is::<Variant>()
             || array.is::<Extension>()
+            || array.is::<Variant>()
     }
 
     fn try_match<'a>(array: &'a dyn DynArray) -> Option<Self::Match<'a>> {