Improve docs for DataFusion integration (#7442)

## Summary

Reworking docs for the DataFusion integration, mostly just more
examples.
Also moved some top-level tests into a dedicated module.

---------

Signed-off-by: Adam Gutglick <adam@spiraldb.com>

Author: AdamGS

docs/user-guide/datafusion.md

@@ -14,7 +14,7 @@ vortex-datafusion = "<version>"
 
 Register the Vortex format with a `SessionContext`:
 
-:::{literalinclude} ../../vortex-datafusion/src/persistent/mod.rs
+:::{literalinclude} ../../vortex-datafusion/src/persistent/tests.rs
 :language: rust
 :dedent:
 :start-after: [setup]
@@ -27,7 +27,7 @@ Register the Vortex format with a `SessionContext`:
 
 Create an external table and query it:
 
-:::{literalinclude} ../../vortex-datafusion/src/persistent/mod.rs
+:::{literalinclude} ../../vortex-datafusion/src/persistent/tests.rs
 :language: rust
 :dedent:
 :start-after: [create]
@@ -49,7 +49,7 @@ You can also register a `ListingTable` directly:
 
 Write query results to Vortex using `INSERT INTO`:
 
-:::{literalinclude} ../../vortex-datafusion/src/persistent/mod.rs
+:::{literalinclude} ../../vortex-datafusion/src/persistent/tests.rs
 :language: rust
 :dedent:
 :start-after: [write]
@@ -63,7 +63,7 @@ partition value.
 
 Filters and projections are pushed down into the Vortex scan:
 
-:::{literalinclude} ../../vortex-datafusion/src/persistent/mod.rs
+:::{literalinclude} ../../vortex-datafusion/src/persistent/tests.rs
 :language: rust
 :dedent:
 :start-after: [query]

vortex-datafusion/src/lib.rs

@@ -1,7 +1,87 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-//! Connectors to enable [DataFusion](https://docs.rs/datafusion/latest/datafusion/) to read [`Vortex`](https://docs.rs/crate/vortex/latest) data.
+//! Integrations between [`Vortex`] and [DataFusion].
+//!
+//! The crate exposes two main entry points:
+//!
+//! - [`VortexFormatFactory`] for the file-based integration used by SQL,
+//!   `CREATE EXTERNAL TABLE`, and
+//!   [`ListingTable`].
+//! - [`v2`] for direct integration from an existing Vortex
+//!   [`DataSourceRef`].
+//!
+//! # Registering The File Format
+//!
+//! Most applications register [`VortexFormatFactory`] with a DataFusion
+//! [`SessionContext`] and then let DataFusion create [`VortexFormat`] and
+//! [`VortexSource`] instances as queries are planned:
+//!
+//! ```no_run
+//! use std::sync::Arc;
+//!
+//! use datafusion::datasource::provider::DefaultTableFactory;
+//! use datafusion::execution::SessionStateBuilder;
+//! use datafusion::prelude::SessionContext;
+//! use datafusion_common::GetExt;
+//! use vortex_datafusion::VortexFormatFactory;
+//!
+//! # #[tokio::main]
+//! # async fn main() -> Result<(), Box<dyn std::error::Error>> {
+//! let factory = Arc::new(VortexFormatFactory::new());
+//! let mut state_builder = SessionStateBuilder::new()
+//!     .with_default_features()
+//!     .with_table_factory(
+//!         factory.get_ext().to_uppercase(),
+//!         Arc::new(DefaultTableFactory::new()),
+//!     );
+//!
+//! if let Some(file_formats) = state_builder.file_formats() {
+//!     file_formats.push(factory.clone() as _);
+//! }
+//!
+//! let ctx = SessionContext::new_with_state(state_builder.build()).enable_url_table();
+//! ctx.sql(
+//!     "CREATE EXTERNAL TABLE metrics (service VARCHAR, value BIGINT) \
+//!      STORED AS vortex LOCATION 'file:///tmp/metrics/'",
+//! )
+//! .await?;
+//! # Ok(())
+//! # }
+//! ```
+//!
+//! # Registering An Existing Vortex Data Source
+//!
+//! If your application already has a Vortex [`DataSourceRef`], use
+//! [`v2::VortexTable`] to register it directly with DataFusion:
+//!
+//! ```no_run
+//! use std::sync::Arc;
+//!
+//! use arrow_schema::Schema;
+//! use datafusion::prelude::SessionContext;
+//! use vortex::VortexSessionDefault;
+//! use vortex::scan::DataSourceRef;
+//! use vortex::session::VortexSession;
+//! use vortex_datafusion::v2::VortexTable;
+//!
+//! # let data_source: DataSourceRef = todo!();
+//! let table = Arc::new(VortexTable::new(
+//!     data_source,
+//!     VortexSession::default(),
+//!     Arc::new(Schema::empty()),
+//! ));
+//!
+//! let ctx = SessionContext::new();
+//! ctx.register_table("vortex_data", table)?;
+//! # Ok::<(), datafusion_common::DataFusionError>(())
+//! ```
+//!
+//! [`Vortex`]: https://docs.rs/crate/vortex/latest
+//! [DataFusion]: https://docs.rs/datafusion/latest/datafusion/
+//! [`ListingTable`]: https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingTable.html
+//! [`DataSourceRef`]: vortex::scan::DataSourceRef
+//! [`SessionContext`]: https://docs.rs/datafusion/latest/datafusion/prelude/struct.SessionContext.html
 #![deny(missing_docs)]
 use std::fmt::Debug;
 
@@ -18,7 +98,11 @@ mod tests;
 pub use convert::exprs::ExpressionConvertor;
 pub use persistent::*;
 
-/// Extension trait to convert our [`Precision`](vortex::stats::Precision) to Datafusion's [`Precision`](datafusion_common::stats::Precision)
+/// Extension trait to convert our [`Precision`] to DataFusion's
+/// [`DataFusionPrecision`].
+///
+/// [`Precision`]: vortex::expr::stats::Precision
+/// [`DataFusionPrecision`]: datafusion_common::stats::Precision
 trait PrecisionExt<T>
 where
     T: Debug + Clone + PartialEq + Eq + PartialOrd,

vortex-datafusion/src/persistent/access_plan.rs

@@ -4,18 +4,46 @@
 use vortex::layout::scan::scan_builder::ScanBuilder;
 use vortex::scan::selection::Selection;
 
-/// Custom Vortex-specific information that can be provided by external indexes or other sources.
+/// Additional Vortex-specific scan constraints attached to a
+/// [`PartitionedFile`].
 ///
-/// This is intended as a low-level interface for users building their own data systems, see the [advance index] example from the DataFusion repo for a similar usage with Parquet.
+/// `VortexAccessPlan` is the hook to use when an external index or planner
+/// already knows that only part of a file needs to be scanned. The plan is
+/// attached as `extensions` on `PartitionedFile`, and the internal
+/// `VortexOpener` applies it before building the Vortex scan.
 ///
-/// [advance index]: https://github.com/apache/datafusion/blob/47df535d2cd5aac5ad5a92bdc837f38e05ea0f0f/datafusion-examples/examples/data_io/parquet_advanced_index.rs
+/// The current access plan surface is intentionally small: it lets callers
+/// provide a [`Selection`] that narrows the rows considered by the scan.
+///
+/// # Example
+///
+/// ```no_run
+/// # use std::sync::Arc;
+/// # use datafusion_datasource::PartitionedFile;
+/// # use vortex::scan::selection::Selection;
+/// use vortex_datafusion::VortexAccessPlan;
+///
+/// # let selection: Selection = todo!();
+/// let file = PartitionedFile::new("metrics.vortex", 1024).with_extensions(Arc::new(
+///     VortexAccessPlan::default().with_selection(selection),
+/// ));
+/// # let _ = file;
+/// ```
+///
+/// This is a low-level integration point for systems building their own access
+/// paths on top of DataFusion. For a conceptually similar Parquet example, see
+/// DataFusion's
+/// [`parquet_advanced_index`].
+///
+/// [`PartitionedFile`]: datafusion_datasource::PartitionedFile
+/// [`parquet_advanced_index`]: https://github.com/apache/datafusion/blob/47df535d2cd5aac5ad5a92bdc837f38e05ea0f0f/datafusion-examples/examples/data_io/parquet_advanced_index.rs
 #[derive(Default)]
 pub struct VortexAccessPlan {
     selection: Option<Selection>,
 }
 
 impl VortexAccessPlan {
-    /// Sets a [`Selection`] for this plan.
+    /// Sets the row [`Selection`] to apply when the file is opened.
     pub fn with_selection(mut self, selection: Selection) -> Self {
         self.selection = Some(selection);
         self
@@ -28,7 +56,10 @@ impl VortexAccessPlan {
         self.selection.as_ref()
     }
 
-    /// Apply the plan to the scan's builder.
+    /// Applies this access plan to a [`ScanBuilder`].
+    ///
+    /// This is used internally by the file opener after it has translated a
+    /// `PartitionedFile` into a Vortex scan.
     pub fn apply_to_builder<A>(&self, mut scan_builder: ScanBuilder<A>) -> ScanBuilder<A>
     where
         A: 'static + Send,