Merge pull request #273 from AdaWorldAPI/claude/lance-4-datafusion-52

feat: bump lance 2→4 + datafusion 51→52 + deltalake 0.30→0.31

Author: AdaWorldAPI

Cargo.lock

@@ -31,11 +31,11 @@ arrow-array = "57"
 arrow-schema = "57"
 arrow-buffer = "57"
 
-# Query engine — DataFusion 51
-datafusion = { version = "51", optional = true }
+# Query engine — DataFusion 52
+datafusion = { version = "52", optional = true }
 
 # Lance columnar format — 100x faster random access than Parquet
-lance = { version = "2.0", optional = true, default-features = false }
+lance = { version = "4", optional = true, default-features = false }
 
 # Async runtime
 tokio = { version = "1.49", features = ["rt-multi-thread", "macros"], optional = true }

crates/holograph/Cargo.toml

@@ -7,7 +7,7 @@ description = "Benchmarks for lance-graph (not published)"
 
 [dependencies]
 lance-graph = { path = "../lance-graph", version = "0.5.3" }
-lance = "2"
+lance = "4"
 arrow-array = "57"
 arrow-schema = "57"
 criterion = { version = "0.5", features = ["async", "async_tokio", "html_reports"] }

crates/lance-graph-benches/Cargo.toml

@@ -12,13 +12,14 @@ lance-graph-contract = { path = "../lance-graph-contract" }
 
 # [persist] — Arrow RecordBatch + Lance dataset ops
 arrow       = { version = "57", optional = true }
-lance       = { version = "2",  optional = true }
+lance       = { version = "4",  optional = true }
 
 # [query] / [query-lite] — DataFusion. Base dep has default-features = false.
 # The full `query` feature adds datafusion/default (compression backends etc).
 # The `query-lite` feature gets logical plan + optimizer without compression,
-# avoiding the xz2/liblzma `links = "lzma"` collision with lance v2.
-datafusion  = { version = "51", optional = true, default-features = false, features = ["unicode_expressions"] }
+# avoiding the xz2/liblzma `links = "lzma"` collision (lance v2 era; may be
+# resolved with lance 4 + datafusion 52 — verify and remove if so).
+datafusion  = { version = "52", optional = true, default-features = false, features = ["unicode_expressions"] }
 
 # [realtime] — version watcher, Phoenix channel shapes, WebSocket
 tokio            = { version = "1", features = ["sync"], optional = true }

crates/lance-graph-callcenter/Cargo.toml

@@ -13,8 +13,8 @@ categories = ["database", "data-structures", "science"]
 [dependencies]
 arrow-schema = "57"
 async-trait = "0.1"
-datafusion = { version = "51", default-features = false }
-lance-namespace = "2"
+datafusion = { version = "52", default-features = false }
+lance-namespace = "4"
 reqwest = { version = "0.12", features = ["json"], optional = true }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"

crates/lance-graph-catalog/Cargo.toml

@@ -3,7 +3,6 @@
 use async_trait::async_trait;
 use lance_namespace::models::{DescribeTableRequest, DescribeTableResponse};
 use lance_namespace::{Error as NamespaceError, LanceNamespace, Result};
-use snafu::location;
 
 /// A namespace that resolves table names relative to a base directory or URI.
 #[derive(Debug, Clone)]
@@ -39,7 +38,6 @@ impl LanceNamespace for DirNamespace {
         let id = request.id.ok_or_else(|| {
             NamespaceError::invalid_input(
                 "DirNamespace requires the table identifier to be provided",
-                location!(),
             )
         })?;
 
@@ -49,7 +47,6 @@ impl LanceNamespace for DirNamespace {
                     "DirNamespace expects identifiers with a single component, got {:?}",
                     id
                 ),
-                location!(),
             ));
         }
 

crates/lance-graph-catalog/src/namespace/directory.rs

@@ -16,7 +16,7 @@ arrow-array = "57"
 arrow-schema = "57"
 chrono = "0.4"
 async-trait = "0.1"
-datafusion = { version = "51", default-features = false, features = [
+datafusion = { version = "52", default-features = false, features = [
     "nested_expressions",
     "regex_expressions",
     "unicode_expressions",
@@ -26,21 +26,21 @@ datafusion = { version = "51", default-features = false, features = [
     "string_expressions",
     "parquet",
 ] }
-datafusion-common = "51"
-datafusion-expr = "51"
-datafusion-sql = "51"
-datafusion-functions-aggregate = "51"
+datafusion-common = "52"
+datafusion-expr = "52"
+datafusion-sql = "52"
+datafusion-functions-aggregate = "52"
 futures = "0.3"
 lance-graph-catalog = { path = "../lance-graph-catalog", version = "0.5.4" }
 lance-graph-contract = { path = "../lance-graph-contract" }
-lance = "2"
-lance-linalg = "2"
-lance-namespace = "2"
+lance = "4"
+lance-linalg = "4"
+lance-namespace = "4"
 nom = "7.1"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 snafu = "0.8"
-deltalake = { version = "0.30", features = ["datafusion", "s3", "azure", "gcs"], optional = true }
+deltalake = { version = "0.31", features = ["datafusion", "s3", "azure", "gcs"], optional = true }
 url = { version = "2", optional = true }
 
 # AdaWorldAPI/ndarray fork with 55 HPC modules (Fingerprint, CAM-PQ, CLAM, BLAS, ZeckF64)
@@ -72,7 +72,7 @@ tensor-codec = ["dep:bgz-tensor"]
 
 [dev-dependencies]
 futures = "0.3"
-lance-arrow = "2"
-lance-index = "2"
+lance-arrow = "4"
+lance-index = "4"
 tempfile = "3"
 tokio = { version = "1.37", features = ["macros", "rt-multi-thread"] }

crates/lance-graph/Cargo.toml

@@ -109,7 +109,16 @@ impl TableReader for DeltaTableReader {
             ))
         })?;
 
-        ctx.register_table(table_name, Arc::new(delta_table))
+        let snapshot = delta_table.snapshot()
+            .map_err(|e| CatalogError::Other(format!("Failed to get Delta snapshot: {e}")))?
+            .snapshot()
+            .clone();
+        let log_store = delta_table.log_store();
+        let scan_config = Default::default();
+        let provider = deltalake::delta_datafusion::DeltaTableProvider::try_new(
+            snapshot, log_store, scan_config,
+        ).map_err(|e| CatalogError::Other(format!("Failed to create DeltaTableProvider: {e}")))?;
+        ctx.register_table(table_name, Arc::new(provider))
             .map_err(|e| {
                 CatalogError::Other(format!(
                     "Failed to register Delta table '{}': {}",