Feature/optimize unnest groupby

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 resolver = "2"
-members = ["datafusion/bio-function-pileup", "datafusion/bio-function-ranges", "datafusion/bio-function-vep"]
+members = ["datafusion/bio-function-pileup", "datafusion/bio-function-ranges", "datafusion/bio-function-vep", "datafusion/bio-function-vcftools", "datafusion/bio-function"]
 exclude = ["datafusion/bio-function-ranges/superintervals"]
 
 [workspace.package]
@@ -15,3 +15,7 @@ datafusion = { version = "50.3.0" }
 tokio = { version = "1.43.0", features = ["rt-multi-thread", "rt", "macros"] }
 futures = "0.3.31"
 log = "0.4.27"
+serial_test = "3.1"
+tempfile = "3.15"
+parquet = "56.1"
+async-trait = "0.1.88"

README.md

@@ -13,8 +13,10 @@ This workspace provides a collection of Rust crates that implement DataFusion UD
 
 | Crate | Description | Status |
 |-------|-------------|--------|
+| **[datafusion-bio-function](datafusion/bio-function)** | Unified bio `SessionContext` setup (`BioSessionExt`, `create_bio_session`) that wires ranges + vcftools optimizations | ✅ |
 | **[datafusion-bio-function-pileup](datafusion/bio-function-pileup)** | Depth-of-coverage (pileup) computation from BAM alignments | ✅ |
 | **[datafusion-bio-function-ranges](datafusion/bio-function-ranges)** | Interval join, coverage, count-overlaps, nearest-neighbor, overlap, merge, cluster, complement, and subtract operations | ✅ |
+| **[datafusion-bio-function-vcftools](datafusion/bio-function-vcftools)** | Fused array transform logical/physical optimization for unnest-transform-array_agg patterns | ✅ |
 | **[datafusion-bio-function-vep](datafusion/bio-function-vep)** | VEP variant annotation via `lookup_variants()` table function with parquet + Fjall KV cache backends | ✅ |
 
 ## Features
@@ -347,7 +349,7 @@ Both schemas include metadata key `bio.coordinate_system_zero_based` (`"true"` o
 Create a bio-configured session and write standard SQL — range overlap joins are automatically optimized:
 
 ```rust
-use datafusion_bio_function_ranges::create_bio_session;
+use datafusion_bio_function::create_bio_session;
 
 #[tokio::main]
 async fn main() -> datafusion::error::Result<()> {

datafusion/bio-function-ranges/README.md

@@ -17,15 +17,15 @@ This crate provides optimized genomic interval operations as DataFusion extensio
 ## Quick Start
 
 ```rust
-use datafusion_bio_function_ranges::{create_bio_session, register_ranges_functions};
+use datafusion_bio_function::{create_bio_session, BioSessionExt};
+use datafusion_bio_function_ranges::{BioConfig, register_ranges_functions};
 
 // Option 1: Create a fully configured session (recommended)
 let ctx = create_bio_session();
 
 // Option 2: Register functions on an existing bio-configured session
 use datafusion::config::ConfigOptions;
 use datafusion::prelude::{SessionConfig, SessionContext};
-use datafusion_bio_function_ranges::{BioConfig, BioSessionExt};
 
 let config = SessionConfig::from(ConfigOptions::new())
     .with_option_extension(BioConfig::default())
@@ -47,18 +47,21 @@ use datafusion_bio_function_ranges::register_ranges_functions;
 register_ranges_functions(&ctx);
 ```
 
-### `create_bio_session()`
+### Unified Bio Session (recommended)
 
-Convenience function that creates a `SessionContext` with:
-- Custom query planner for automatic interval join detection
-- Physical optimizer rule that converts hash/nested-loop joins to interval joins
+Use `datafusion-bio-function` to create a `SessionContext` with:
+- interval join optimization (from ranges)
+- fused array transform optimization (from vcftools)
 - `BioConfig` extension for algorithm selection via `SET bio.*` statements
-- All SQL table functions: `coverage()`, `count_overlaps()`, `nearest()`, `overlap()`, `merge()`, `cluster()`, `complement()`, `subtract()`
+
+Then register ranges table functions from this crate.
 
 ```rust
-use datafusion_bio_function_ranges::create_bio_session;
+use datafusion_bio_function::create_bio_session;
+use datafusion_bio_function_ranges::register_ranges_functions;
 
 let ctx = create_bio_session();
+register_ranges_functions(&ctx);
 ```
 
 ## SQL Table Functions
@@ -218,7 +221,7 @@ SELECT * FROM subtract('left_table', 'right_table', 'l_chr', 'l_s', 'l_e', 'r_ch
 
 ## Interval Join (SQL)
 
-When using a bio-configured session (`create_bio_session()` or `BioSessionExt::new_with_bio()`), SQL joins with range overlap conditions are automatically optimized:
+When using a bio-configured session (`datafusion_bio_function::create_bio_session()` or `BioSessionExt::new_with_bio()` from `datafusion-bio-function`), SQL joins with range overlap conditions are automatically optimized:
 
 ```sql
 -- Automatically detected and optimized as interval join
@@ -287,10 +290,10 @@ This crate replaces the `sequila-core` crate from the [sequila-native](https://g
 
 | sequila-native | datafusion-bio-function-ranges |
 |----------------|-------------------------------|
-| `sequila_core::session_context::SeQuiLaSessionExt` | `BioSessionExt` |
+| `sequila_core::session_context::SeQuiLaSessionExt` | `datafusion_bio_function::BioSessionExt` |
 | `sequila_core::session_context::SequilaConfig` | `BioConfig` |
 | `sequila_core::session_context::Algorithm` | `Algorithm` |
-| `SessionContext::new_with_sequila(config)` | `SessionContext::new_with_bio(config)` |
+| `SessionContext::new_with_sequila(config)` | `SessionContext::new_with_bio(config)` (from `datafusion-bio-function`) |
 
 ### Configuration Namespace
 
@@ -317,18 +320,18 @@ let ctx = SessionContext::new_with_sequila(config);
 
 **After (datafusion-bio-function-ranges):**
 ```rust
-use datafusion_bio_function_ranges::{create_bio_session, register_ranges_functions};
+use datafusion_bio_function::{create_bio_session, BioSessionExt};
+use datafusion_bio_function_ranges::{BioConfig, register_ranges_functions};
 
 // Simple: creates context with everything configured
 let ctx = create_bio_session();
+register_ranges_functions(&ctx);
 
 // Or manually:
-use datafusion_bio_function_ranges::{BioConfig, BioSessionExt};
-
 let config = SessionConfig::from(options)
     .with_option_extension(BioConfig::default());
 let ctx = SessionContext::new_with_bio(config);
-register_ranges_functions(&ctx);  // registers coverage() and count_overlaps() UDTFs
+register_ranges_functions(&ctx);  // registers ranges table functions
 ```
 
 ### New SQL Table Functions

datafusion/bio-function-ranges/src/algorithms.rs

@@ -0,0 +1,77 @@
+use datafusion::config::ConfigField;
+use std::str::FromStr;
+
+#[derive(Debug, Eq, PartialEq, Default, Clone, Copy)]
+pub enum Algorithm {
+    #[default]
+    Coitrees,
+    IntervalTree,
+    ArrayIntervalTree,
+    Lapper,
+    SuperIntervals,
+    CoitreesNearest,
+    CoitreesCountOverlaps,
+}
+
+#[derive(Debug)]
+pub struct ParseAlgorithmError(String);
+
+impl std::fmt::Display for ParseAlgorithmError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
+impl std::error::Error for ParseAlgorithmError {}
+
+impl FromStr for Algorithm {
+    type Err = ParseAlgorithmError;
+
+    #[inline]
+    fn from_str(s: &str) -> Result<Algorithm, Self::Err> {
+        match s.to_lowercase().as_str() {
+            "coitrees" => Ok(Algorithm::Coitrees),
+            "intervaltree" => Ok(Algorithm::IntervalTree),
+            "arrayintervaltree" => Ok(Algorithm::ArrayIntervalTree),
+            "lapper" => Ok(Algorithm::Lapper),
+            "superintervals" => Ok(Algorithm::SuperIntervals),
+            "coitreesnearest" => Ok(Algorithm::CoitreesNearest),
+            "coitreescountoverlaps" => Ok(Algorithm::CoitreesCountOverlaps),
+            _ => Err(ParseAlgorithmError(format!(
+                "Can't parse '{s}' as Algorithm"
+            ))),
+        }
+    }
+}
+
+impl std::fmt::Display for Algorithm {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let val = match self {
+            Algorithm::Coitrees => "Coitrees",
+            Algorithm::IntervalTree => "IntervalTree",
+            Algorithm::ArrayIntervalTree => "ArrayIntervalTree",
+            Algorithm::Lapper => "Lapper",
+            Algorithm::SuperIntervals => "SuperIntervals",
+            Algorithm::CoitreesNearest => "CoitreesNearest",
+            Algorithm::CoitreesCountOverlaps => "CoitreesCountOverlaps",
+        };
+        write!(f, "{val}")
+    }
+}
+
+impl From<ParseAlgorithmError> for datafusion::error::DataFusionError {
+    fn from(e: ParseAlgorithmError) -> Self {
+        datafusion::error::DataFusionError::External(Box::new(e))
+    }
+}
+
+impl ConfigField for Algorithm {
+    fn set(&mut self, _key: &str, value: &str) -> datafusion::common::Result<()> {
+        *self = value.parse::<Algorithm>()?;
+        Ok(())
+    }
+
+    fn visit<V: datafusion::config::Visit>(&self, visitor: &mut V, name: &str, doc: &'static str) {
+        visitor.some(name, self, doc)
+    }
+}

datafusion/bio-function-ranges/src/config.rs

@@ -0,0 +1,16 @@
+use datafusion::common::extensions_options;
+use datafusion::config::ConfigExtension;
+
+use crate::algorithms::Algorithm;
+
+extensions_options! {
+    pub struct BioConfig {
+        pub prefer_interval_join: bool, default = true
+        pub interval_join_algorithm: Algorithm, default = Algorithm::default()
+        pub interval_join_low_memory: bool, default = false
+    }
+}
+
+impl ConfigExtension for BioConfig {
+    const PREFIX: &'static str = "bio";
+}

datafusion/bio-function-ranges/src/lib.rs

@@ -1,6 +1,8 @@
+pub mod algorithms;
 pub mod array_utils;
 pub mod cluster;
 pub mod complement;
+pub mod config;
 pub mod count_overlaps;
 pub mod filter_op;
 pub mod grouped_stream;
@@ -10,21 +12,23 @@ pub mod nearest;
 pub mod nearest_index;
 pub mod overlap;
 pub mod physical_planner;
-pub mod session_context;
+pub mod session_builder;
 pub mod subtract;
 pub mod table_function;
 
 // Re-export key types
+pub use algorithms::Algorithm;
 pub use cluster::ClusterProvider;
 pub use complement::ComplementProvider;
+pub use config::BioConfig;
 pub use count_overlaps::CountOverlapsProvider;
 pub use filter_op::FilterOp;
 pub use merge::MergeProvider;
 pub use nearest::NearestProvider;
 pub use overlap::OverlapProvider;
-pub use physical_planner::BioQueryPlanner;
 pub use physical_planner::IntervalJoinPhysicalOptimizationRule;
+pub use physical_planner::RangesQueryPlanner;
 pub use physical_planner::joins::interval_join::IntervalJoinExec;
-pub use session_context::{Algorithm, BioConfig, BioSessionExt, create_bio_session};
+pub use session_builder::{create_ranges_session, create_ranges_session_with_config};
 pub use subtract::SubtractProvider;
 pub use table_function::register_ranges_functions;