More tests and docs

Signed-off-by: Adam Gutglick <adam@spiraldb.com>

Author: AdamGS

encodings/parquet-variant/src/array.rs

@@ -23,16 +23,18 @@ use vortex_error::VortexResult;
 use vortex_error::vortex_ensure;
 use vortex_mask::Mask;
 
-/// An array encoding that stores variant data in the Parquet Variant binary format.
+/// Array storage for Arrow's canonical `arrow.parquet.variant` extension type.
 ///
-/// Contains up to three children following the Arrow Parquet Variant canonical extension type:
-/// - `metadata` (always present): binary array with variant type information
-/// - `value` (optional): binary array with un-shredded serialized variant values
-/// - `typed_value` (optional): array of any type with shredded column data
+/// `ParquetVariantArray` preserves semi-structured data stored as Parquet Variant values in a
+/// lossless form and supports both unshredded and shredded layouts. Its storage matches the
+/// canonical extension type contract:
+/// - `metadata` is always present and non-nullable
+/// - `value` stores unshredded variant bytes when present
+/// - `typed_value` stores shredded data when present
 ///
-/// At least one of `value` or `typed_value` must be present.
-/// The `typed_value` supports full recursive shredding — it can be a primitive, list, or struct
-/// where nested struct/list elements themselves contain value/typed_value children.
+/// At least one of `value` or `typed_value` must be present. `typed_value` may be a primitive,
+/// list, or struct, with nested shredded children following the same recursive rules as the
+/// Arrow canonical extension type docs.
 #[derive(Clone, Debug)]
 pub struct ParquetVariantArray {
     pub(crate) dtype: DType,
@@ -44,7 +46,7 @@ pub struct ParquetVariantArray {
 }
 
 impl ParquetVariantArray {
-    /// Creates a new ParquetVariantArray.
+    /// Creates a Parquet Variant array that follows the canonical extension storage layout.
     pub fn try_new(
         metadata: ArrayRef,
         value: Option<ArrayRef>,
@@ -53,7 +55,7 @@ impl ParquetVariantArray {
         Self::try_new_with_validity(Validity::AllValid, metadata, value, typed_value)
     }
 
-    /// Creates a new ParquetVariantArray with explicit parent validity.
+    /// Creates a Parquet Variant array with explicit outer validity.
     pub fn try_new_with_validity(
         validity: Validity,
         metadata: ArrayRef,

encodings/parquet-variant/src/lib.rs

@@ -1,21 +1,28 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-//! This crate exposes a Vortex encoding that supports variant arrays, encoded as parquet's
-//! [Variant encoding], in order to allow for zero-copy export to Arrow's new
-//! [canonical extension type].
+//! Vortex support for Arrow's canonical `arrow.parquet.variant` extension type.
 //!
-//! The encoding follows the Arrow Parquet Variant canonical extension type structure:
-//! - `metadata` (binary, required): type information for arrays/objects, field names and offsets
-//! - `value` (binary, optional): un-shredded serialized variant values
-//! - `typed_value` (any type, optional): shredded column data with a known type
+//! This encoding provides a lossless representation of semi-structured data stored as
+//! [Parquet Variant values] inside Arrow columns. It also supports [shredded variant values],
+//! allowing systems to pass variant-encoded data around without special handling unless they
+//! need to inspect the encoded contents directly.
 //!
-//! At least one of `value` or `typed_value` must be present. The `typed_value` child supports
-//! full recursive shredding — it can be a primitive type, a list (whose elements are variant
-//! nodes with value/typed_value), or a struct (whose fields are variant nodes).
+//! The storage type is a `Struct` that follows the Arrow canonical extension contract:
+//! - `metadata` (required): a non-nullable binary child containing variant metadata
+//! - `value` (optional): a binary child containing unshredded variant bytes
+//! - `typed_value` (optional): a shredded child with a primitive, list, or struct layout
 //!
-//! [Variant encoding]: https://parquet.apache.org/docs/file-format/types/variantencoding/
-//! [canonical extension type]: https://arrow.apache.org/docs/format/CanonicalExtensions.html#parquet-variant
+//! At least one of `value` or `typed_value` must be present. Nested shredded values recurse
+//! through the same `value` and `typed_value` structure described by the canonical extension
+//! type documentation.
+//!
+//! See the Arrow canonical extension docs for the storage rules, and the Parquet format
+//! specification for the binary representation.
+//!
+//! [Parquet Variant values]: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md
+//! [shredded variant values]: https://github.com/apache/parquet-format/blob/master/VariantShredding.md
+//! [Arrow canonical extension type]: https://arrow.apache.org/docs/format/CanonicalExtensions.html#parquet-variant
 
 mod array;
 mod operations;