Improve docs for DataFusion integration

Signed-off-by: Adam Gutglick <adam@spiraldb.com>

Author: AdamGS

vortex-datafusion/src/lib.rs

@@ -1,7 +1,87 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-//! Connectors to enable [DataFusion](https://docs.rs/datafusion/latest/datafusion/) to read [`Vortex`](https://docs.rs/crate/vortex/latest) data.
+//! Integrations between [`Vortex`] and [DataFusion].
+//!
+//! The crate exposes two main entry points:
+//!
+//! - [`VortexFormatFactory`] for the file-based integration used by SQL,
+//!   `CREATE EXTERNAL TABLE`, and
+//!   [`ListingTable`].
+//! - [`v2`] for direct integration from an existing Vortex
+//!   [`DataSourceRef`].
+//!
+//! # Registering The File Format
+//!
+//! Most applications register [`VortexFormatFactory`] with a DataFusion
+//! [`SessionContext`] and then let DataFusion create [`VortexFormat`] and
+//! [`VortexSource`] instances as queries are planned:
+//!
+//! ```no_run
+//! use std::sync::Arc;
+//!
+//! use datafusion::datasource::provider::DefaultTableFactory;
+//! use datafusion::execution::SessionStateBuilder;
+//! use datafusion::prelude::SessionContext;
+//! use datafusion_common::GetExt;
+//! use vortex_datafusion::VortexFormatFactory;
+//!
+//! # #[tokio::main]
+//! # async fn main() -> Result<(), Box<dyn std::error::Error>> {
+//! let factory = Arc::new(VortexFormatFactory::new());
+//! let mut state_builder = SessionStateBuilder::new()
+//!     .with_default_features()
+//!     .with_table_factory(
+//!         factory.get_ext().to_uppercase(),
+//!         Arc::new(DefaultTableFactory::new()),
+//!     );
+//!
+//! if let Some(file_formats) = state_builder.file_formats() {
+//!     file_formats.push(factory.clone() as _);
+//! }
+//!
+//! let ctx = SessionContext::new_with_state(state_builder.build()).enable_url_table();
+//! ctx.sql(
+//!     "CREATE EXTERNAL TABLE metrics (service VARCHAR, value BIGINT) \
+//!      STORED AS vortex LOCATION 'file:///tmp/metrics/'",
+//! )
+//! .await?;
+//! # Ok(())
+//! # }
+//! ```
+//!
+//! # Registering An Existing Vortex Data Source
+//!
+//! If your application already has a Vortex [`DataSourceRef`], use
+//! [`v2::VortexTable`] to register it directly with DataFusion:
+//!
+//! ```no_run
+//! use std::sync::Arc;
+//!
+//! use arrow_schema::Schema;
+//! use datafusion::prelude::SessionContext;
+//! use vortex::VortexSessionDefault;
+//! use vortex::scan::DataSourceRef;
+//! use vortex::session::VortexSession;
+//! use vortex_datafusion::v2::VortexTable;
+//!
+//! # let data_source: DataSourceRef = todo!();
+//! let table = Arc::new(VortexTable::new(
+//!     data_source,
+//!     VortexSession::default(),
+//!     Arc::new(Schema::empty()),
+//! ));
+//!
+//! let ctx = SessionContext::new();
+//! ctx.register_table("vortex_data", table)?;
+//! # Ok::<(), datafusion_common::DataFusionError>(())
+//! ```
+//!
+//! [`Vortex`]: https://docs.rs/crate/vortex/latest
+//! [DataFusion]: https://docs.rs/datafusion/latest/datafusion/
+//! [`ListingTable`]: https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingTable.html
+//! [`DataSourceRef`]: vortex::scan::DataSourceRef
+//! [`SessionContext`]: https://docs.rs/datafusion/latest/datafusion/prelude/struct.SessionContext.html
 #![deny(missing_docs)]
 use std::fmt::Debug;
 
@@ -18,7 +98,11 @@ mod tests;
 pub use convert::exprs::ExpressionConvertor;
 pub use persistent::*;
 
-/// Extension trait to convert our [`Precision`](vortex::stats::Precision) to Datafusion's [`Precision`](datafusion_common::stats::Precision)
+/// Extension trait to convert our [`Precision`] to DataFusion's
+/// [`DataFusionPrecision`].
+///
+/// [`Precision`]: vortex::expr::stats::Precision
+/// [`DataFusionPrecision`]: datafusion_common::stats::Precision
 trait PrecisionExt<T>
 where
     T: Debug + Clone + PartialEq + Eq + PartialOrd,

vortex-datafusion/src/persistent/access_plan.rs

@@ -4,18 +4,46 @@
 use vortex::layout::scan::scan_builder::ScanBuilder;
 use vortex::scan::selection::Selection;
 
-/// Custom Vortex-specific information that can be provided by external indexes or other sources.
+/// Additional Vortex-specific scan constraints attached to a
+/// [`PartitionedFile`].
 ///
-/// This is intended as a low-level interface for users building their own data systems, see the [advance index] example from the DataFusion repo for a similar usage with Parquet.
+/// `VortexAccessPlan` is the hook to use when an external index or planner
+/// already knows that only part of a file needs to be scanned. The plan is
+/// attached as `extensions` on `PartitionedFile`, and the internal
+/// `VortexOpener` applies it before building the Vortex scan.
 ///
-/// [advance index]: https://github.com/apache/datafusion/blob/47df535d2cd5aac5ad5a92bdc837f38e05ea0f0f/datafusion-examples/examples/data_io/parquet_advanced_index.rs
+/// The current access plan surface is intentionally small: it lets callers
+/// provide a [`Selection`] that narrows the rows considered by the scan.
+///
+/// # Example
+///
+/// ```no_run
+/// # use std::sync::Arc;
+/// # use datafusion_datasource::PartitionedFile;
+/// # use vortex::scan::selection::Selection;
+/// use vortex_datafusion::VortexAccessPlan;
+///
+/// # let selection: Selection = todo!();
+/// let file = PartitionedFile::new("metrics.vortex", 1024).with_extensions(Arc::new(
+///     VortexAccessPlan::default().with_selection(selection),
+/// ));
+/// # let _ = file;
+/// ```
+///
+/// This is a low-level integration point for systems building their own access
+/// paths on top of DataFusion. For a conceptually similar Parquet example, see
+/// DataFusion's
+/// [`parquet_advanced_index`].
+///
+/// [`PartitionedFile`]: datafusion_datasource::PartitionedFile
+/// [`parquet_advanced_index`]: https://github.com/apache/datafusion/blob/47df535d2cd5aac5ad5a92bdc837f38e05ea0f0f/datafusion-examples/examples/data_io/parquet_advanced_index.rs
 #[derive(Default)]
 pub struct VortexAccessPlan {
     selection: Option<Selection>,
 }
 
 impl VortexAccessPlan {
-    /// Sets a [`Selection`] for this plan.
+    /// Sets the row [`Selection`] to apply when the file is opened.
     pub fn with_selection(mut self, selection: Selection) -> Self {
         self.selection = Some(selection);
         self
@@ -28,7 +56,10 @@ impl VortexAccessPlan {
         self.selection.as_ref()
     }
 
-    /// Apply the plan to the scan's builder.
+    /// Applies this access plan to a [`ScanBuilder`].
+    ///
+    /// This is used internally by the file opener after it has translated a
+    /// `PartitionedFile` into a Vortex scan.
     pub fn apply_to_builder<A>(&self, mut scan_builder: ScanBuilder<A>) -> ScanBuilder<A>
     where
         A: 'static + Send,

vortex-datafusion/src/persistent/format.rs

@@ -70,7 +70,53 @@ use crate::convert::TryToDataFusion;
 
 const DEFAULT_FOOTER_INITIAL_READ_SIZE_BYTES: usize = MAX_POSTSCRIPT_SIZE as usize + EOF_SIZE;
 
-/// Vortex implementation of a DataFusion [`FileFormat`].
+/// DataFusion [`FileFormat`] implementation for `.vortex` files.
+///
+/// Most applications do not construct `VortexFormat` directly. Instead, they
+/// register [`VortexFormatFactory`] with a [`SessionContext`] and let
+/// DataFusion instantiate `VortexFormat` as tables are planned.
+///
+/// Construct `VortexFormat` directly when you are wiring a [`ListingTable`] by
+/// hand and need to pass a file format into [`ListingOptions`].
+///
+/// # Example
+///
+/// ```no_run
+/// use std::sync::Arc;
+///
+/// use datafusion::datasource::listing::ListingOptions;
+/// use datafusion::datasource::listing::ListingTable;
+/// use datafusion::datasource::listing::ListingTableConfig;
+/// use datafusion::datasource::listing::ListingTableUrl;
+/// use datafusion::prelude::SessionContext;
+/// use tempfile::tempdir;
+/// use vortex::VortexSessionDefault;
+/// use vortex::session::VortexSession;
+/// use vortex_datafusion::VortexFormat;
+///
+/// # #[tokio::main]
+/// # async fn main() -> Result<(), Box<dyn std::error::Error>> {
+/// let ctx = SessionContext::new();
+/// let dir = tempdir()?;
+///
+/// let format = Arc::new(VortexFormat::new(VortexSession::default()));
+/// let table_url = ListingTableUrl::parse(dir.path().to_str().unwrap())?;
+/// let config = ListingTableConfig::new(table_url)
+///     .with_listing_options(
+///         ListingOptions::new(format).with_session_config_options(ctx.state().config()),
+///     )
+///     .infer_schema(&ctx.state())
+///     .await?;
+///
+/// let table = ListingTable::try_new(config)?;
+/// # let _ = table;
+/// # Ok(())
+/// # }
+/// ```
+///
+/// [`SessionContext`]: https://docs.rs/datafusion/latest/datafusion/prelude/struct.SessionContext.html
+/// [`ListingTable`]: https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingTable.html
+/// [`ListingOptions`]: https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingOptions.html
 pub struct VortexFormat {
     session: VortexSession,
     opts: VortexTableOptions,
@@ -85,9 +131,24 @@ impl Debug for VortexFormat {
 }
 
 config_namespace! {
-    /// Options to configure the [`VortexFormat`].
+    /// Options to configure [`VortexFormat`] and [`VortexSource`].
     ///
-    /// Can be set through a DataFusion [`SessionConfig`].
+    /// These options are usually set on a [`VortexFormatFactory`] and inherited
+    /// by the `VortexFormat` / `VortexSource` instances created for individual
+    /// tables.
+    ///
+    /// # Example
+    ///
+    /// ```rust
+    /// use vortex_datafusion::{VortexFormatFactory, VortexTableOptions};
+    ///
+    /// let factory = VortexFormatFactory::new().with_options(VortexTableOptions {
+    ///     projection_pushdown: true,
+    ///     scan_concurrency: Some(8),
+    ///     ..Default::default()
+    /// });
+    /// # let _ = factory;
+    /// ```
     ///
     /// [`SessionConfig`]: https://docs.rs/datafusion/latest/datafusion/prelude/struct.SessionConfig.html
     pub struct VortexTableOptions {
@@ -113,7 +174,44 @@ config_namespace! {
 
 impl Eq for VortexTableOptions {}
 
-/// Minimal factory to create [`VortexFormat`] instances.
+/// Registration entry point for the file-backed Vortex integration.
+///
+/// `VortexFormatFactory` is the type most applications use. Register it with a
+/// DataFusion session, and DataFusion will create [`VortexFormat`] values for
+/// `CREATE EXTERNAL TABLE`, [`ListingTable`], and URL-table scans.
+///
+/// The factory stores a [`VortexSession`] and default [`VortexTableOptions`].
+/// Those defaults are copied into the formats and sources created for each
+/// table.
+///
+/// # Example
+///
+/// ```no_run
+/// use std::sync::Arc;
+///
+/// use datafusion::datasource::provider::DefaultTableFactory;
+/// use datafusion::execution::SessionStateBuilder;
+/// use datafusion_common::GetExt;
+/// use vortex_datafusion::{VortexFormatFactory, VortexTableOptions};
+///
+/// let factory = Arc::new(VortexFormatFactory::new().with_options(VortexTableOptions {
+///     projection_pushdown: true,
+///     ..Default::default()
+/// }));
+///
+/// let mut state_builder = SessionStateBuilder::new()
+///     .with_default_features()
+///     .with_table_factory(
+///         factory.get_ext().to_uppercase(),
+///         Arc::new(DefaultTableFactory::new()),
+///     );
+///
+/// if let Some(file_formats) = state_builder.file_formats() {
+///     file_formats.push(factory.clone() as _);
+/// }
+/// ```
+///
+/// [`ListingTable`]: https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingTable.html
 #[derive(Debug)]
 pub struct VortexFormatFactory {
     session: VortexSession,
@@ -127,7 +225,7 @@ impl GetExt for VortexFormatFactory {
 }
 
 impl VortexFormatFactory {
-    /// Creates a new instance with a default [`VortexSession`] and default options.
+    /// Creates a factory with a default [`VortexSession`] and default options.
     #[expect(
         clippy::new_without_default,
         reason = "FormatFactory defines `default` method, so having `Default` implementation is confusing"
@@ -139,23 +237,33 @@ impl VortexFormatFactory {
         }
     }
 
-    /// Creates a new instance with customized session and default options for all [`VortexFormat`] instances created from this factory.
+    /// Creates a factory with an explicit session and default options.
     ///
-    /// The options can be overridden by table-level configuration pass in [`FileFormatFactory::create`].
+    /// The supplied options become the baseline for every [`VortexFormat`]
+    /// created by this factory. DataFusion may still override them with
+    /// table-level options passed into [`FileFormatFactory::create`].
     pub fn new_with_options(session: VortexSession, options: VortexTableOptions) -> Self {
         Self {
             session,
             options: Some(options),
         }
     }
 
-    /// Override the default options for this factory.
+    /// Overrides the default options for this factory.
+    ///
+    /// This is the usual way to turn on features such as projection pushdown for
+    /// every table created through the factory.
+    ///
+    /// # Example
     ///
-    /// For example:
     /// ```rust
     /// use vortex_datafusion::{VortexFormatFactory, VortexTableOptions};
     ///
-    /// let factory = VortexFormatFactory::new().with_options(VortexTableOptions::default());
+    /// let factory = VortexFormatFactory::new().with_options(VortexTableOptions {
+    ///     projection_pushdown: true,
+    ///     ..Default::default()
+    /// });
+    /// # let _ = factory;
     /// ```
     pub fn with_options(mut self, options: VortexTableOptions) -> Self {
         self.options = Some(options);
@@ -195,17 +303,23 @@ impl FileFormatFactory for VortexFormatFactory {
 }
 
 impl VortexFormat {
-    /// Create a new instance with default options.
+    /// Creates a format with default [`VortexTableOptions`].
+    ///
+    /// Prefer [`VortexFormatFactory`] when registering with a session. Construct
+    /// `VortexFormat` directly when building [`ListingOptions`] manually.
+    ///
+    /// [`ListingOptions`]: https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingOptions.html
     pub fn new(session: VortexSession) -> Self {
         Self::new_with_options(session, VortexTableOptions::default())
     }
 
-    /// Creates a new instance with configured by a [`VortexTableOptions`].
+    /// Creates a format with explicit [`VortexTableOptions`].
     pub fn new_with_options(session: VortexSession, opts: VortexTableOptions) -> Self {
         Self { session, opts }
     }
 
-    /// Return the format specific configuration
+    /// Returns the format-specific configuration that will be copied into the
+    /// [`VortexSource`] created for a scan.
     pub fn options(&self) -> &VortexTableOptions {
         &self.opts
     }