refactor: De-generic executor and refactor ORM stack (#349)

* refactor: streamline catalog state and execution caches

* refactor: reuse table codec buffers

* refactor: introduce metadata arenas for execution state

* refactor: normalize catalog metadata arenas

* refactor: use static evaluator dispatch refs

* feat: show column refs in describe output

* refactor: replace bincode and serde serialization

* perf: reduce execution allocations and shell dependencies

* perf: reserve column pruning removed positions

* feat: make optional dependencies feature-gated

* refactor: make binder consume owned statements

* refactor: isolate parser and ORM binding APIs

* refactor: bind ORM queries through binder plans

* refactor: de-generic executor to reduce compile time

Remove the transaction type parameter from executor nodes and route execution state through a runtime trait so executor code is compiled once instead of monomorphized per storage backend.

* test: update runtime test helpers

* fix: expose wasm ddl and analyze APIs

* fix: resolve CI failures after executor de-generic refactor

* refactor: unify execution context cache handling

* fix: avoid cloning prepared statements during binding

* Reduce projection and tuple serialization allocations

* Update TPCC results and clean clippy warnings

Author: KKould

Cargo.lock

@@ -2,7 +2,7 @@
 
 [package]
 name          = "kite_sql"
-version       = "0.2.1"
+version       = "0.3.0"
 edition       = "2021"
 build         = "build.rs"
 authors       = ["Kould <kould2333@gmail.com>", "Xwg <loloxwg@gmail.com>"]
@@ -13,32 +13,30 @@ repository    = "https://github.com/KipData/KiteSQL"
 readme        = "README.md"
 keywords      = ["sql", "sqlite", "database", "mysql"]
 categories    = ["development-tools", "database"]
-default-run   = "kite_sql"
-
-[[bin]]
-name              = "kite_sql"
-path              = "src/bin/server.rs"
-required-features = ["net", "rocksdb"]
 
 [[bin]]
 name              = "kitesql-shell"
 path              = "src/bin/shell.rs"
-required-features = ["rocksdb"]
+required-features = ["rocksdb", "shell"]
 
 [lib]
 doctest = false
 crate-type = ["cdylib", "rlib"]
 
 [features]
-default = ["macros", "rocksdb"]
+default = ["time", "macros", "parser", "rocksdb"]
+time    = ["dep:chrono"]
+copy    = ["dep:csv"]
+decimal = ["dep:rust_decimal"]
 macros  = []
-orm     = []
+orm     = ["macros"]
+parser  = ["dep:sqlparser"]
 rocksdb = ["dep:rocksdb"]
 unsafe_txdb_checkpoint = ["rocksdb", "dep:librocksdb-sys"]
 lmdb    = ["dep:lmdb", "dep:lmdb-sys"]
-net     = ["rocksdb", "dep:pgwire", "dep:async-trait", "dep:clap", "dep:env_logger", "dep:futures", "dep:log", "dep:tokio"]
 pprof   = ["pprof/criterion", "pprof/flamegraph"]
-python  = ["dep:pyo3"]
+python  = ["parser", "dep:pyo3"]
+shell   = ["parser", "dep:comfy-table", "dep:rustyline"]
 
 [[bench]]
 name    = "query_bench"
@@ -48,36 +46,24 @@ required-features = ["pprof"]
 
 [dependencies]
 ahash                 = { version = "0.8" }
-bincode               = { version = "1" }
-bumpalo               = { version = "3", features = ["allocator-api2", "collections", "std"] }
+bumpalo               = { version = "3", default-features = false, features = ["collections"] }
 byteorder             = { version = "1" }
-chrono                = { version = "0.4" }
-comfy-table           = { version = "7", default-features = false }
-csv                   = { version = "1" }
 fixedbitset           = { version = "0.4" }
 itertools             = { version = "0.12" }
-ordered-float         = { version = "4", features = ["serde"] }
+ordered-float         = { version = "4" }
 paste                 = { version = "1" }
-parking_lot           = { version = "0.12", features = ["arc_lock"] }
-pyo3                  = { version = "0.23", features = ["auto-initialize"], optional = true }
 recursive             = { version = "0.1" }
-rust_decimal          = { version = "1" }
-serde                 = { version = "1", features = ["derive", "rc"] }
 kite_sql_serde_macros = { version = "0.2.0", path = "kite_sql_serde_macros" }
-siphasher             = { version = "1", features = ["serde"] }
-sqlparser             = { version = "0.61", default-features = false, features = ["std"] }
-thiserror             = { version = "1" }
-ulid                  = { version = "1", features = ["serde"] }
-
-# Feature: net
-async-trait           = { version = "0.1", optional = true }
-clap                  = { version = "4.5", features = ["derive"], optional = true }
-env_logger            = { version = "0.11", optional = true }
-futures               = { version = "0.3", optional = true }
-log                   = { version = "0.4", optional = true }
-pgwire                = { version = "0.28.0", optional = true }
-tokio                 = { version = "1.36", features = ["full"], optional = true }
+siphasher             = { version = "1" }
+ulid                  = { version = "1" }
 
+# Optional dependencies for features
+comfy-table           = { version = "7", default-features = false, optional = true }
+chrono                = { version = "0.4", optional = true }
+csv                   = { version = "1", optional = true }
+pyo3                  = { version = "0.23", features = ["auto-initialize"], optional = true }
+rust_decimal          = { version = "1", default-features = false, features = ["std"], optional = true }
+sqlparser             = { version = "0.61", default-features = false, features = ["std"], optional = true }
 
 [target.'cfg(unix)'.dev-dependencies]
 pprof = { version = "0.15", features = ["flamegraph", "criterion"] }
@@ -90,11 +76,12 @@ tempfile              = { version = "3.10" }
 sqlite                = { version = "0.34" }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-rocksdb = { version = "0.23", optional = true, default-features = false, features = ["bindgen-runtime"] }
+# Optional dependencies for features
 librocksdb-sys = { version = "0.17.1", optional = true }
 lmdb = { version = "0.8.0", optional = true }
 lmdb-sys = { version = "0.8.0", optional = true }
-rustyline = { version = "14", default-features = false }
+rocksdb = { version = "0.23", optional = true, default-features = false, features = ["bindgen-runtime"] }
+rustyline = { version = "14", default-features = false, optional = true }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 wasm-bindgen        = { version = "0.2.106" }
@@ -106,7 +93,6 @@ base64              = { version = "0.21" }
 getrandom           = { version = "0.2", features = ["js"] }
 getrandom_03        = { package = "getrandom", version = "0.3", features = ["wasm_js"] }
 js-sys              = { version = "0.3.83" }
-serde-wasm-bindgen  = { version = "0.6.5" }
 once_cell           = { version = "1" }
 
 [target.'cfg(target_arch = "wasm32")'.dev-dependencies]

Cargo.toml

@@ -18,7 +18,7 @@ test:
 
 ## Run Python binding API tests implemented with pyo3.
 test-python:
-	PYO3_PYTHON=$(PYO3_PYTHON) $(CARGO) test --features python test_python_
+	PYO3_PYTHON=$(PYO3_PYTHON) $(CARGO) test --features python,decimal test_python_
 
 ## Perform a `cargo check` across the workspace.
 cargo-check:

Makefile

@@ -55,7 +55,8 @@ For the full ORM guide, see [`src/orm/README.md`](src/orm/README.md).
 ```rust
 use kite_sql::db::DataBaseBuilder;
 use kite_sql::errors::DatabaseError;
-use kite_sql::{Model, Projection};
+use kite_sql::orm::OrmQueryResultExt;
+use kite_sql::Model;
 
 #[derive(Default, Debug, PartialEq, Model)]
 #[model(table = "users")]
@@ -71,15 +72,8 @@ struct User {
     age: Option<i32>,
 }
 
-#[derive(Default, Debug, PartialEq, Projection)]
-struct UserSummary {
-    id: i32,
-    #[projection(rename = "user_name")]
-    display_name: String,
-}
-
 fn main() -> Result<(), DatabaseError> {
-    let database = DataBaseBuilder::path("./data").build_rocksdb()?;
+    let mut database = DataBaseBuilder::path("./data").build_rocksdb()?;
     // Or: let database = DataBaseBuilder::path("./data").build_lmdb()?;
 
     database.migrate::<User>()?;
@@ -100,24 +94,31 @@ fn main() -> Result<(), DatabaseError> {
     ])?;
 
     database
-        .from::<User>()
-        .eq(User::id(), 1)
-        .update()
-        .set(User::age(), Some(19))
-        .execute()?;
+        .bind(|ctx| {
+            ctx.mutate::<User>()?
+                .filter(|e| e.column(User::id())?.eq(1))?
+                .update(|u| u.set_value(User::age(), Some(19)))
+        })?
+        .done()?;
 
     database
-        .from::<User>()
-        .eq(User::id(), 2)
-        .delete()?;
+        .bind(|ctx| {
+            ctx.mutate::<User>()?
+                .filter(|e| e.column(User::id())?.eq(2))?
+                .delete()
+        })?
+        .done()?;
 
     let users = database
-        .from::<User>()
-        .gte(User::age(), 18)
-        .project::<UserSummary>()
-        .asc(User::name())
-        .limit(10)
-        .fetch()?;
+        .bind(|ctx| {
+            ctx.from::<User>()?
+                .filter(|e| e.column(User::age())?.gte(18))?
+                .project_scalars((User::id(), User::name()))?
+                .order_by(User::name())?
+                .limit(10)?
+                .finish()
+        })?
+        .project_tuple::<(i32, String)>();
 
     for user in users {
         println!("{:?}", user?);
@@ -138,6 +139,7 @@ fn main() -> Result<(), DatabaseError> {
 - Transaction isolation is documented in [`docs/transaction-isolation.md`](docs/transaction-isolation.md).
 - Cargo features:
   - `rocksdb` is enabled by default
+  - `parser` is enabled by default and provides the SQL parser frontend
   - `lmdb` is optional
   - `unsafe_txdb_checkpoint` enables experimental checkpoint support for RocksDB `TransactionDB`
   - `cargo check --no-default-features --features lmdb` builds an LMDB-only native configuration
@@ -164,7 +166,7 @@ Checkpoint support and feature-gating details are documented in [docs/features.m
 import { WasmDatabase } from "./pkg/kite_sql.js";
 
 const db = new WasmDatabase();
-await db.execute("create table demo(id int primary key, v int)");
+await db.ddl("create table demo(id int primary key, v int)");
 await db.execute("insert into demo values (1, 2), (2, 4)");
 const rows = db.run("select * from demo").rows();
 console.log(rows.map((r) => r.values.map((v) => v.Int32 ?? v)));
@@ -198,10 +200,10 @@ Recent 720-second local comparison on the machine above:
 
 | Backend | TpmC | New-Order p90 | Payment p90 | Order-Status p90 | Delivery p90 | Stock-Level p90 |
 | --- | ---: | ---: | ---: | ---: | ---: | ---: |
-| KiteSQL LMDB | 68394 | 0.001s | 0.001s | 0.001s | 0.002s | 0.001s |
-| KiteSQL RocksDB | 30387 | 0.001s | 0.001s | 0.001s | 0.015s | 0.002s |
-| SQLite balanced | 41690 | 0.001s | 0.001s | 0.001s | 0.001s | 0.001s |
-| SQLite practical | 38861 | 0.001s | 0.001s | 0.001s | 0.001s | 0.001s |
+| KiteSQL LMDB | 61723 | 0.001s | 0.001s | 0.001s | 0.002s | 0.001s |
+| KiteSQL RocksDB | 30446 | 0.001s | 0.001s | 0.001s | 0.016s | 0.002s |
+| SQLite balanced | 42989 | 0.001s | 0.001s | 0.001s | 0.001s | 0.001s |
+| SQLite practical | 42276 | 0.001s | 0.001s | 0.001s | 0.001s | 0.001s |
 
 The detailed raw outputs are recorded in [tpcc/README.md](tpcc/README.md).
 #### 👉[check more](tpcc/README.md)

README.md

@@ -47,7 +47,7 @@ mod app {
         pub c2: String,
     }
 
-    fn run_with_database<S: Storage>(database: Database<S>) -> Result<(), DatabaseError> {
+    fn run_with_database<S: Storage>(mut database: Database<S>) -> Result<(), DatabaseError> {
         database.create_table_if_not_exists::<MyStruct>()?;
         database.insert(&MyStruct {
             c1: 0,
@@ -63,22 +63,25 @@ mod app {
         })?;
 
         database
-            .from::<MyStruct>()
-            .eq(MyStruct::c1(), 1)
-            .update()
-            .set(MyStruct::c2(), "ONE")
-            .execute()?;
-        database.from::<MyStruct>().eq(MyStruct::c1(), 2).delete()?;
+            .bind(|ctx| {
+                ctx.mutate::<MyStruct>()?
+                    .filter(|e| e.column(MyStruct::c1())?.eq(1))?
+                    .update(|u| u.set_value(MyStruct::c2(), "ONE"))
+            })?
+            .done()?;
+        database
+            .bind(|ctx| {
+                ctx.mutate::<MyStruct>()?
+                    .filter(|e| e.column(MyStruct::c1())?.eq(2))?
+                    .delete()
+            })?
+            .done()?;
 
         for row in database.fetch::<MyStruct>()? {
             println!("{:?}", row?);
         }
 
-        let mut agg = database.run("select count(*) from my_struct")?;
-        if let Some(count_row) = agg.next() {
-            println!("row count = {:?}", count_row?);
-        }
-        agg.done()?;
+        println!("row count = {}", database.fetch::<MyStruct>()?.count());
 
         database.drop_table::<MyStruct>()?;
 
@@ -87,23 +90,39 @@ mod app {
 
     pub fn run() -> Result<(), DatabaseError> {
         reset_example_dir()?;
-        let backend = env::var("KITESQL_BACKEND").unwrap_or_else(|_| "rocksdb".to_string());
+        let backend = env::var("KITESQL_BACKEND").unwrap_or_else(|_| {
+            #[cfg(feature = "rocksdb")]
+            {
+                "rocksdb".to_string()
+            }
+            #[cfg(all(not(feature = "rocksdb"), feature = "lmdb"))]
+            {
+                "lmdb".to_string()
+            }
+            #[cfg(all(not(feature = "rocksdb"), not(feature = "lmdb")))]
+            {
+                "memory".to_string()
+            }
+        });
 
         match backend.to_ascii_lowercase().as_str() {
+            "memory" => {
+                run_with_database(DataBaseBuilder::path(EXAMPLE_DB_PATH).build_in_memory()?)
+            }
             #[cfg(feature = "rocksdb")]
             "rocksdb" => run_with_database(DataBaseBuilder::path(EXAMPLE_DB_PATH).build_rocksdb()?),
             #[cfg(feature = "lmdb")]
             "lmdb" => run_with_database(DataBaseBuilder::path(EXAMPLE_DB_PATH).build_lmdb()?),
             other => Err(DatabaseError::InvalidValue(format!(
                 "unsupported example backend '{other}', expected {}",
-                {
-                    let mut expected = Vec::new();
+                [
+                    "memory",
                     #[cfg(feature = "rocksdb")]
-                    expected.push("rocksdb");
+                    "rocksdb",
                     #[cfg(feature = "lmdb")]
-                    expected.push("lmdb");
-                    expected.join(" or ")
-                }
+                    "lmdb",
+                ]
+                .join(" or ")
             ))),
         }
     }

examples/hello_world.rs

@@ -41,10 +41,8 @@ mod app {
     pub fn run() -> Result<(), DatabaseError> {
         reset_example_dir()?;
         // Optimistic transactions are currently backed by RocksDB.
-        let database = DataBaseBuilder::path(EXAMPLE_DB_PATH).build_optimistic()?;
-        database
-            .run("create table if not exists t1 (c1 int primary key, c2 int)")?
-            .done()?;
+        let mut database = DataBaseBuilder::path(EXAMPLE_DB_PATH).build_optimistic()?;
+        database.ddl("create table if not exists t1 (c1 int primary key, c2 int)")?;
         let mut transaction = database.new_transaction()?;
 
         transaction
@@ -65,6 +63,7 @@ mod app {
             Tuple::new(None, vec![DataValue::Int32(1), DataValue::Int32(1)])
         );
         assert!(iter.next().is_none());
+        iter.done()?;
 
         let mut tx2 = database.new_transaction()?;
         tx2.run("update t1 set c2 = 99 where c1 = 0")?.done()?;
@@ -79,7 +78,7 @@ mod app {
         );
         drop(tx2);
 
-        database.run("drop table t1")?.done()?;
+        database.ddl("drop table t1")?;
 
         Ok(())
     }

examples/transaction.rs

@@ -8,8 +8,8 @@ const { WasmDatabase } = require("../pkg/kite_sql.js");
 async function main() {
   const db = new WasmDatabase();
 
-  await db.execute("drop table if exists my_struct");
-  await db.execute("create table my_struct (c1 int primary key, c2 int)");
+  await db.ddl("drop table if exists my_struct");
+  await db.ddl("create table my_struct (c1 int primary key, c2 int)");
   await db.execute("insert into my_struct values(0, 0), (1, 1)");
 
   const iter = db.run("select * from my_struct");
@@ -48,7 +48,7 @@ async function main() {
     [1, 11],
   ]);
 
-  await db.execute("drop table my_struct");
+  await db.ddl("drop table my_struct");
   console.log("wasm hello_world test passed");
 }