Expanding conceptual docs (and some other minor docs) (#8552)

## Summary

Adds more docs, both module level conceptual docs which pulls from
`docs/`, and many local changes documenting specific functions or types.
Also tried to include basic examples.

The main changes are in `vortex`, `vortex-array` and `vortex-file` and
`vortex-layout`.

Was done with the help of Claude to try and find gaps/mistakes, and fill
in some of the obvious changes.

---------

Signed-off-by: Adam Gutglick <adam@spiraldb.com>

Author: AdamGS

encodings/fastlanes/Cargo.toml

@@ -13,6 +13,9 @@ repository = { workspace = true }
 rust-version = { workspace = true }
 version = { workspace = true }
 
+[package.metadata.docs.rs]
+all-features = true
+
 [lints]
 workspace = true
 

encodings/fastlanes/src/lib.rs

@@ -3,6 +3,29 @@
 
 #![expect(clippy::cast_possible_truncation)]
 
+//! FastLanes integer encodings for Vortex arrays.
+//!
+//! This crate provides SIMD-friendly integer encodings:
+//!
+//! - [`BitPacked`] stores fixed-width integer values using the minimum bit width plus optional
+//!   patches.
+//! - [`FoR`] stores frame-of-reference deltas from a base value.
+//! - [`Delta`] stores adjacent deltas in chunked form.
+//! - [`RLE`] stores repeated runs.
+//!
+//! Call [`initialize`] to register the encodings and encoding-specific aggregate kernels in a
+//! session before deserializing or executing arrays that may contain these encodings.
+//!
+//! ```rust
+//! let session = vortex_array::array_session();
+//! vortex_fastlanes::initialize(&session);
+//! ```
+//!
+//! ## Paper
+//!
+//! The original encodings are described in the paper [The FastLanes Compression Layout](https://15721.courses.cs.cmu.edu/spring2024/papers/03-data2/p2132-afroozeh.pdf),
+//! but are not fully binary compatible. See the underlying [fastlanes](https://github.com/spiraldb/fastlanes) crate for more details.
+
 pub use bitpacking::*;
 pub use delta::*;
 pub use r#for::*;

encodings/pco/src/array.rs

@@ -282,6 +282,7 @@ pub(crate) fn vortex_err_from_pco(err: PcoError) -> VortexError {
 }
 
 #[derive(Clone, Debug)]
+/// Pco array encoding marker.
 pub struct Pco;
 
 impl Pco {
@@ -317,6 +318,7 @@ pub(super) const NUM_SLOTS: usize = 1;
 pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"];
 
 #[derive(Clone, Debug)]
+/// Encoding-specific data for a [`PcoArray`].
 pub struct PcoData {
     pub(crate) chunk_metas: Vec<ByteBuffer>,
     pub(crate) pages: Vec<ByteBuffer>,
@@ -338,6 +340,7 @@ impl Display for PcoData {
 }
 
 impl PcoData {
+    /// Validate dtype, validity, slice, and Pco component invariants.
     pub fn validate(&self, dtype: &DType, len: usize, validity: &Validity) -> VortexResult<()> {
         let _ = number_type_from_ptype(self.ptype);
         vortex_ensure!(
@@ -391,6 +394,7 @@ impl PcoData {
         Ok(())
     }
 
+    /// Construct unsliced Pco data from chunk metadata, pages, and serialized metadata.
     pub fn new(
         chunk_metas: Vec<ByteBuffer>,
         pages: Vec<ByteBuffer>,
@@ -409,6 +413,7 @@ impl PcoData {
         }
     }
 
+    /// Compress a primitive array into Pco data.
     pub fn from_primitive(
         parray: ArrayView<'_, Primitive>,
         level: usize,
@@ -497,6 +502,11 @@ impl PcoData {
         ))
     }
 
+    /// Downcast and compress an array into Pco data.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if the input is not a primitive array or compression fails.
     pub fn from_array(
         array: ArrayRef,
         level: usize,
@@ -512,6 +522,7 @@ impl PcoData {
         Self::from_primitive(parray.as_view(), level, nums_per_page, ctx)
     }
 
+    /// Decompress this Pco data into a primitive array.
     pub fn decompress(
         &self,
         unsliced_validity: &Validity,

encodings/pco/src/lib.rs

@@ -1,6 +1,23 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+//! Pco-backed numeric compression encoding for Vortex arrays.
+//!
+//! [`PcoArray`] stores valid primitive numeric values in Pco chunks and pages, while Vortex
+//! validity tracks null rows separately. Page metadata lets slices decompress only the components
+//! required for the requested row range.
+//!
+//! Pco supports integer and floating-point primitive dtypes handled by the upstream `pco` crate.
+//! It is normally selected through the BtrBlocks compressor when the `pco` feature is enabled.
+//! To deserialize arrays manually, register the encoding in the array session:
+//!
+//! ```rust
+//! use vortex_array::session::ArraySessionExt;
+//!
+//! let session = vortex_array::array_session();
+//! session.arrays().register(vortex_pco::Pco);
+//! ```
+
 mod array;
 mod compute;
 mod rules;
@@ -9,27 +26,34 @@ mod slice;
 pub use array::*;
 
 #[derive(Clone, prost::Message)]
+/// Metadata for one Pco page.
 pub struct PcoPageInfo {
     // Since pco limits to 2^24 values per chunk, u32 is sufficient for the
     // count of values.
+    /// Number of valid primitive values stored in this page.
     #[prost(uint32, tag = "1")]
     pub n_values: u32,
 }
 
 // We're calling this Info instead of Metadata because ChunkMeta refers to a specific
 // component of a Pco file.
 #[derive(Clone, prost::Message)]
+/// Metadata for one Pco chunk.
 pub struct PcoChunkInfo {
+    /// Pages contained in this chunk.
     #[prost(message, repeated, tag = "1")]
     pub pages: Vec<PcoPageInfo>,
 }
 
 #[derive(Clone, prost::Message)]
+/// Serialized metadata for a [`PcoArray`].
 pub struct PcoMetadata {
     // would be nice to reuse one header per vortex file, but it's really only 1 byte, so
     // no issue duplicating it here per PcoArray
+    /// Pco file header bytes.
     #[prost(bytes, tag = "1")]
     pub header: Vec<u8>,
+    /// Metadata for each compressed chunk.
     #[prost(message, repeated, tag = "2")]
     pub chunks: Vec<PcoChunkInfo>,
 }

encodings/zstd/Cargo.toml

@@ -13,6 +13,9 @@ repository = { workspace = true }
 rust-version = { workspace = true }
 version = { workspace = true }
 
+[package.metadata.docs.rs]
+all-features = true
+
 [lints]
 workspace = true
 

encodings/zstd/src/array.rs

@@ -252,9 +252,11 @@ impl VTable for Zstd {
 }
 
 #[derive(Clone, Debug)]
+/// Zstd array encoding marker.
 pub struct Zstd;
 
 impl Zstd {
+    /// Construct a [`ZstdArray`] from validated compressed data and validity.
     pub fn try_new(dtype: DType, data: ZstdData, validity: Validity) -> VortexResult<ZstdArray> {
         let len = data.len();
         data.validate(&dtype, len, &validity)?;
@@ -309,6 +311,7 @@ impl Zstd {
         )
     }
 
+    /// Decompress a [`ZstdArray`] into its canonical Vortex representation.
     pub fn decompress(array: &ZstdArray, ctx: &mut ExecutionCtx) -> VortexResult<ArrayRef> {
         let unsliced_validity = child_to_validity(
             array.as_ref().slots()[0].as_ref(),
@@ -325,6 +328,7 @@ pub(super) const NUM_SLOTS: usize = 1;
 pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"];
 
 #[derive(Clone, Debug)]
+/// Encoding-specific data for a [`ZstdArray`].
 pub struct ZstdData {
     pub(crate) dictionary: Option<ByteBuffer>,
     pub(crate) frames: Vec<ByteBuffer>,
@@ -344,13 +348,21 @@ impl Display for ZstdData {
     }
 }
 
+/// Movable parts of a [`ZstdData`] value plus its validity.
 pub struct ZstdDataParts {
+    /// Optional zstd dictionary shared by all frames.
     pub dictionary: Option<ByteBuffer>,
+    /// Compressed zstd frames.
     pub frames: Vec<ByteBuffer>,
+    /// Serialized frame and dictionary metadata.
     pub metadata: ZstdMetadata,
+    /// Unsliced validity for the array.
     pub validity: Validity,
+    /// Unsliced row count.
     pub n_rows: usize,
+    /// Start of this logical slice in unsliced row coordinates.
     pub slice_start: usize,
+    /// End of this logical slice in unsliced row coordinates.
     pub slice_stop: usize,
 }
 
@@ -466,6 +478,7 @@ pub fn reconstruct_views(
 }
 
 impl ZstdData {
+    /// Construct unsliced zstd data from raw frames and metadata.
     pub fn new(
         dictionary: Option<ByteBuffer>,
         frames: Vec<ByteBuffer>,
@@ -482,6 +495,7 @@ impl ZstdData {
         }
     }
 
+    /// Validate dtype, slice, validity, frame, and dictionary invariants.
     pub fn validate(&self, dtype: &DType, len: usize, validity: &Validity) -> VortexResult<()> {
         vortex_ensure!(
             matches!(
@@ -796,6 +810,9 @@ impl ZstdData {
         Ok(ZstdData::new(dictionary, frames, metadata, vbv.len()))
     }
 
+    /// Compress a supported canonical array into zstd data.
+    ///
+    /// Returns `Ok(None)` for canonical variants that this encoding does not support.
     pub fn from_canonical(
         canonical: &Canonical,
         level: i32,
@@ -819,6 +836,11 @@ impl ZstdData {
         }
     }
 
+    /// Canonicalize and compress an array into zstd data.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if the array's canonical form is unsupported or compression fails.
     pub fn from_array(
         array: ArrayRef,
         level: i32,
@@ -1019,6 +1041,7 @@ impl ZstdData {
         self.slice_stop == self.slice_start
     }
 
+    /// Split this data into movable parts, attaching the supplied validity.
     pub fn into_parts(self, validity: Validity) -> ZstdDataParts {
         ZstdDataParts {
             dictionary: self.dictionary,

encodings/zstd/src/lib.rs

@@ -1,6 +1,26 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+//! Zstd-backed compression encodings for variable-width Vortex arrays.
+//!
+//! [`ZstdArray`] stores UTF-8 or binary values as one or more zstd frames, optionally sharing a
+//! trained dictionary across frames. Frame metadata lets slices decompress only the frames that can
+//! contribute values to the requested row range.
+//!
+//! With the `unstable_encodings` feature, `ZstdBuffers` stores the buffers of another encoding as
+//! independently compressed zstd buffers while preserving the inner encoding metadata.
+//!
+//! This crate exposes array encodings only. Compression scheme selection is wired through
+//! `vortex-btrblocks` and file writing. To deserialize arrays manually, register the encoding in the
+//! array session:
+//!
+//! ```rust
+//! use vortex_array::session::ArraySessionExt;
+//!
+//! let session = vortex_array::array_session();
+//! session.arrays().register(vortex_zstd::Zstd);
+//! ```
+
 pub use array::*;
 #[cfg(feature = "unstable_encodings")]
 pub use zstd_buffers::*;
@@ -16,28 +36,38 @@ mod zstd_buffers;
 mod test;
 
 #[derive(Clone, prost::Message)]
+/// Metadata for one zstd frame.
 pub struct ZstdFrameMetadata {
+    /// Uncompressed byte size of this frame.
     #[prost(uint64, tag = "1")]
     pub uncompressed_size: u64,
+    /// Number of valid values stored in this frame.
     #[prost(uint64, tag = "2")]
     pub n_values: u64,
 }
 
 #[derive(Clone, prost::Message)]
+/// Serialized metadata for a [`ZstdArray`].
 pub struct ZstdMetadata {
     // optional, will be 0 if there's no dictionary
+    /// Dictionary size in bytes, or `0` when no dictionary is present.
     #[prost(uint32, tag = "1")]
     pub dictionary_size: u32,
+    /// Metadata for each compressed frame.
     #[prost(message, repeated, tag = "2")]
     pub frames: Vec<ZstdFrameMetadata>,
 }
 
 #[derive(Clone, prost::Message)]
+/// Serialized metadata for the unstable `ZstdBuffers` encoding.
 pub struct ZstdBuffersMetadata {
+    /// Encoding id of the inner array whose buffers were compressed.
     #[prost(string, tag = "1")]
     pub inner_encoding_id: String,
+    /// Serialized metadata of the inner array.
     #[prost(bytes = "vec", tag = "2")]
     pub inner_metadata: Vec<u8>,
+    /// Uncompressed byte size of each compressed buffer.
     #[prost(uint64, repeated, tag = "3")]
     pub uncompressed_sizes: Vec<u64>,
     /// Alignment of each buffer in bytes (must be a power of two).