Skip to content

Commit a21a7e6

Browse files
farhan-syahfarhan-syah
committed
feat: fully remove DataFusion from all crates
DataFusion is completely removed from the workspace — not a single crate depends on it anymore. nodedb (Origin): - Delete all 26 UDF files + sequence/ + wasm UDF impls (~4500 lines) - Delete datafusion_pool from nodedb-mem - Delete ts_udfs from nodedb-query - Replace eval.rs with sqlparser-based constant expression evaluator - Replace validate.rs with sqlparser syntax check - Remove DataFusion from Cargo.toml nodedb-lite (Embedded): - Rewrite query engine to use nodedb-sql instead of DataFusion - Add LiteCatalog implementing SqlCatalog trait - Delete table_provider.rs, strict_provider.rs, columnar_provider.rs, arrow_convert.rs, spatial_udf.rs, prepared.rs - Replace datafusion::arrow with arrow crate - Remove DataFusion from Cargo.toml Workspace: - Remove all DataFusion workspace dependencies from root Cargo.toml - Zero datafusion references remain in any Cargo.toml
1 parent 4c7bdf4 commit a21a7e6

70 files changed

Lines changed: 544 additions & 8411 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Cargo.toml‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -133,14 +133,6 @@ tonic = "0.14"
133133
# Arrow (columnar format)
134134
arrow = { version = "58", default-features = false, features = ["ipc"] }
135135

136-
# Query engine (retained for procedural executor + function body validation)
137-
datafusion = { version = "53", default-features = false, features = ["sql"] }
138-
datafusion-common = "53"
139-
datafusion-execution = "53"
140-
datafusion-expr = "53"
141-
datafusion-physical-plan = "53"
142-
datafusion-functions-window-common = "53"
143-
144136
# Config & serialization
145137
toml = "0.8"
146138
anyhow = "1"

‎nodedb-lite/Cargo.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ sonic-rs = { workspace = true }
3535
loro = { workspace = true }
3636
wasmtime = { workspace = true }
3737
redb = { workspace = true }
38-
datafusion = { workspace = true }
38+
nodedb-sql = { workspace = true }
39+
arrow = { workspace = true }
3940
crc32c = { workspace = true }
4041
aes-gcm = { workspace = true }
4142
argon2 = { workspace = true }

‎nodedb-lite/src/engine/strict/arrow.rs‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,26 +5,26 @@ use std::sync::Arc;
55
use nodedb_types::columnar::{ColumnType, StrictSchema};
66

77
/// Convert an Arrow `DataType` from a `ColumnType` (for schema construction).
8-
pub fn column_type_to_arrow(ct: &ColumnType) -> datafusion::arrow::datatypes::DataType {
8+
pub fn column_type_to_arrow(ct: &ColumnType) -> arrow::datatypes::DataType {
99
match ct {
10-
ColumnType::Int64 => datafusion::arrow::datatypes::DataType::Int64,
11-
ColumnType::Float64 => datafusion::arrow::datatypes::DataType::Float64,
12-
ColumnType::String => datafusion::arrow::datatypes::DataType::Utf8,
13-
ColumnType::Bool => datafusion::arrow::datatypes::DataType::Boolean,
14-
ColumnType::Bytes | ColumnType::Geometry => datafusion::arrow::datatypes::DataType::Binary,
15-
ColumnType::Timestamp => datafusion::arrow::datatypes::DataType::Timestamp(
16-
datafusion::arrow::datatypes::TimeUnit::Microsecond,
10+
ColumnType::Int64 => arrow::datatypes::DataType::Int64,
11+
ColumnType::Float64 => arrow::datatypes::DataType::Float64,
12+
ColumnType::String => arrow::datatypes::DataType::Utf8,
13+
ColumnType::Bool => arrow::datatypes::DataType::Boolean,
14+
ColumnType::Bytes | ColumnType::Geometry => arrow::datatypes::DataType::Binary,
15+
ColumnType::Timestamp => arrow::datatypes::DataType::Timestamp(
16+
arrow::datatypes::TimeUnit::Microsecond,
1717
None,
1818
),
19-
ColumnType::Decimal => datafusion::arrow::datatypes::DataType::Utf8, // Lossless string representation
20-
ColumnType::Uuid => datafusion::arrow::datatypes::DataType::Utf8,
21-
ColumnType::Vector(_) => datafusion::arrow::datatypes::DataType::Binary, // Packed f32 bytes
19+
ColumnType::Decimal => arrow::datatypes::DataType::Utf8, // Lossless string representation
20+
ColumnType::Uuid => arrow::datatypes::DataType::Utf8,
21+
ColumnType::Vector(_) => arrow::datatypes::DataType::Binary, // Packed f32 bytes
2222
}
2323
}
2424

2525
/// Build an Arrow schema from a StrictSchema (for DataFusion table registration).
26-
pub fn strict_schema_to_arrow(schema: &StrictSchema) -> datafusion::arrow::datatypes::SchemaRef {
27-
use datafusion::arrow::datatypes::{Field, Schema};
26+
pub fn strict_schema_to_arrow(schema: &StrictSchema) -> arrow::datatypes::SchemaRef {
27+
use arrow::datatypes::{Field, Schema};
2828
let fields: Vec<Field> = schema
2929
.columns
3030
.iter()

‎nodedb-lite/src/engine/strict/crud.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,7 @@ impl<S: StorageEngine> StrictEngine<S> {
265265
&self,
266266
collection: &str,
267267
col_idx: usize,
268-
) -> Result<datafusion::arrow::array::ArrayRef, LiteError> {
268+
) -> Result<arrow::array::ArrayRef, LiteError> {
269269
let state = self.get_state(collection)?;
270270
let tuples = self.scan_raw(collection).await?;
271271
let refs: Vec<&[u8]> = tuples.iter().map(|t| t.as_slice()).collect();
@@ -279,7 +279,7 @@ impl<S: StorageEngine> StrictEngine<S> {
279279
&self,
280280
collection: &str,
281281
col_indices: &[usize],
282-
) -> Result<Vec<datafusion::arrow::array::ArrayRef>, LiteError> {
282+
) -> Result<Vec<arrow::array::ArrayRef>, LiteError> {
283283
let state = self.get_state(collection)?;
284284
let tuples = self.scan_raw(collection).await?;
285285
let refs: Vec<&[u8]> = tuples.iter().map(|t| t.as_slice()).collect();

‎nodedb-lite/src/query/arrow_convert.rs‎

Lines changed: 0 additions & 87 deletions
This file was deleted.

‎nodedb-lite/src/query/catalog.rs‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
//! SqlCatalog implementation for Lite.
2+
//!
3+
//! Resolves collection metadata from the CRDT, strict, and columnar engines.
4+
5+
use std::sync::{Arc, Mutex};
6+
7+
use nodedb_sql::types::*;
8+
9+
use crate::engine::columnar::ColumnarEngine;
10+
use crate::engine::crdt::CrdtEngine;
11+
use crate::engine::strict::StrictEngine;
12+
use crate::storage::engine::StorageEngine;
13+
14+
/// Catalog adapter for Lite that resolves collections from local engines.
15+
pub struct LiteCatalog<S: StorageEngine> {
16+
crdt: Arc<Mutex<CrdtEngine>>,
17+
strict: Arc<Mutex<StrictEngine<S>>>,
18+
columnar: Arc<Mutex<ColumnarEngine<S>>>,
19+
}
20+
21+
impl<S: StorageEngine> LiteCatalog<S> {
22+
pub fn new(
23+
crdt: Arc<Mutex<CrdtEngine>>,
24+
strict: Arc<Mutex<StrictEngine<S>>>,
25+
columnar: Arc<Mutex<ColumnarEngine<S>>>,
26+
) -> Self {
27+
Self {
28+
crdt,
29+
strict,
30+
columnar,
31+
}
32+
}
33+
}
34+
35+
impl<S: StorageEngine> SqlCatalog for LiteCatalog<S> {
36+
fn get_collection(&self, name: &str) -> Option<CollectionInfo> {
37+
// Check strict collections first.
38+
if let Ok(strict) = self.strict.lock() {
39+
if let Some(schema) = strict.schema(name) {
40+
let columns = schema
41+
.columns
42+
.iter()
43+
.map(|c| ColumnInfo {
44+
name: c.name.clone(),
45+
data_type: convert_column_type(&c.column_type),
46+
nullable: c.nullable,
47+
is_primary_key: c.primary_key,
48+
})
49+
.collect();
50+
let pk = schema
51+
.columns
52+
.iter()
53+
.find(|c| c.primary_key)
54+
.map(|c| c.name.clone());
55+
return Some(CollectionInfo {
56+
name: name.into(),
57+
engine: EngineType::DocumentStrict,
58+
columns,
59+
primary_key: pk,
60+
has_auto_tier: false,
61+
});
62+
}
63+
}
64+
65+
// Check columnar collections.
66+
if let Ok(columnar) = self.columnar.lock() {
67+
if columnar.schema(name).is_some() {
68+
return Some(CollectionInfo {
69+
name: name.into(),
70+
engine: EngineType::Columnar,
71+
columns: Vec::new(),
72+
primary_key: None,
73+
has_auto_tier: false,
74+
});
75+
}
76+
}
77+
78+
// Check CRDT (schemaless) collections.
79+
if let Ok(crdt) = self.crdt.lock() {
80+
if crdt.collection_names().iter().any(|n| n == name) {
81+
return Some(CollectionInfo {
82+
name: name.into(),
83+
engine: EngineType::DocumentSchemaless,
84+
columns: vec![ColumnInfo {
85+
name: "id".into(),
86+
data_type: SqlDataType::String,
87+
nullable: false,
88+
is_primary_key: true,
89+
}],
90+
primary_key: Some("id".into()),
91+
has_auto_tier: false,
92+
});
93+
}
94+
}
95+
96+
None
97+
}
98+
}
99+
100+
fn convert_column_type(ct: &nodedb_types::columnar::ColumnType) -> SqlDataType {
101+
use nodedb_types::columnar::ColumnType;
102+
match ct {
103+
ColumnType::Int64 => SqlDataType::Int64,
104+
ColumnType::Float64 => SqlDataType::Float64,
105+
ColumnType::String => SqlDataType::String,
106+
ColumnType::Bool => SqlDataType::Bool,
107+
ColumnType::Bytes | ColumnType::Geometry => SqlDataType::Bytes,
108+
ColumnType::Timestamp => SqlDataType::Timestamp,
109+
ColumnType::Decimal | ColumnType::Uuid => SqlDataType::String,
110+
ColumnType::Vector(dim) => SqlDataType::Vector(*dim as usize),
111+
}
112+
}

0 commit comments

Comments
 (0)