refactor: improve execution pipeline and TPCC backends (#313)

* refactor: optimizer normalization passes

* refactor: normalization scheduling to avoid matcher scans

* chore: output_columns reduces the number of Vecs constructed

* refactor: derive column positions during binding instead of BindPosition fallback

* refactor: avoid redundant operator clones in normalization rules

* feat: add LMDB storage and fix scalar subquery semantics

- add LMDB storage backend
- restore scalar subqueries in WHERE to join-aware binding
- enforce scalar subquery cardinality at execution time
- return NULL for empty scalar subqueries and error on multi-row results

* refactor: borrow range bounds and reuse delete buffers

* refactor: make key encoding paths borrowed and reusable

* refactor: storage buffers and trim optimizer column-reference overhead

* refactor: column pruning to reuse borrowed column summaries

* refactor: remove memo and choose physical impls directly

* refactor: optimizer hint propagation and preindex implementation rules

* refactor: optimizer annotate pass to merge post-rules and restore borrowed scan hints

* optimizer: avoid schema diff in column pruning remap

* optimizer: avoid recursive expr walks in column pruning

* perf: prebind range comparison evaluators

* refactior: new ExecArena-based executor model

* fix: tpcc segfault caused by bump-backed executor drop order

* refactor: reuse arena result tuple across execution pipeline

* chore: uncheck for decode tuple primary key when query

* refactor(tpcc): split backends and improve benchmark tooling

* refactor(execution): unify executor node API and trim test helpers

* chore: codefmt

* refactor(stats): load sketches alongside histograms in StatisticsMeta

* feat: improve normalization rules and add tpcc benchmark runner

* chore(release): bump kite_sql and macros crate to 0.2.0

* feat(cargo): gate native backends behind rocksdb/lmdb features

Author: KKould

.gitignore

@@ -25,4 +25,4 @@ kite_sql_tpcc
 copy.csv
 
 tests/data/row_20000.csv
-tests/data/distinct_rows.csv
+tests/data/distinct_rows.csv

Cargo.lock

@@ -2,7 +2,7 @@
 
 [package]
 name          = "kite_sql"
-version       = "0.1.8"
+version       = "0.2.0"
 edition       = "2021"
 authors       = ["Kould <kould2333@gmail.com>", "Xwg <loloxwg@gmail.com>"]
 description   = "SQL as a Function for Rust"
@@ -17,17 +17,19 @@ default-run   = "kite_sql"
 [[bin]]
 name              = "kite_sql"
 path              = "src/bin/server.rs"
-required-features = ["net"]
+required-features = ["net", "rocksdb"]
 
 [lib]
 doctest = false
 crate-type = ["cdylib", "rlib"]
 
 [features]
-default = ["macros"]
+default = ["macros", "rocksdb"]
 macros  = []
 orm     = []
-net     = ["dep:pgwire", "dep:async-trait", "dep:clap", "dep:env_logger", "dep:futures", "dep:log", "dep:tokio"]
+rocksdb = ["dep:rocksdb"]
+lmdb    = ["dep:lmdb", "dep:lmdb-sys"]
+net     = ["rocksdb", "dep:pgwire", "dep:async-trait", "dep:clap", "dep:env_logger", "dep:futures", "dep:log", "dep:tokio"]
 pprof   = ["pprof/criterion", "pprof/flamegraph"]
 python  = ["dep:pyo3"]
 
@@ -55,13 +57,12 @@ recursive             = { version = "0.1" }
 regex                 = { version = "1" }
 rust_decimal          = { version = "1" }
 serde                 = { version = "1", features = ["derive", "rc"] }
-kite_sql_serde_macros = { version = "0.1.2", path = "kite_sql_serde_macros" }
+kite_sql_serde_macros = { version = "0.2.0", path = "kite_sql_serde_macros" }
 siphasher             = { version = "1", features = ["serde"] }
 sqlparser             = { version = "0.61", features = ["serde"] }
 thiserror             = { version = "1" }
 typetag               = { version = "0.2" }
 ulid                  = { version = "1", features = ["serde"] }
-genawaiter            = { version = "0.99" }
 
 # Feature: net
 async-trait           = { version = "0.1", optional = true }
@@ -84,7 +85,9 @@ tempfile              = { version = "3.10" }
 sqlite                = { version = "0.34" }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-rocksdb = { version = "0.23" }
+rocksdb = { version = "0.23", optional = true }
+lmdb = { version = "0.8.0", optional = true }
+lmdb-sys = { version = "0.8.0", optional = true }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 wasm-bindgen        = { version = "0.2.106" }

Cargo.toml

@@ -3,8 +3,12 @@ CARGO ?= cargo
 WASM_PACK ?= wasm-pack
 SQLLOGIC_PATH ?= tests/slt/**/*.slt
 PYO3_PYTHON ?= /usr/bin/python3.12
+TPCC_MEASURE_TIME ?= 15
+TPCC_NUM_WARE ?= 1
+TPCC_PPROF_OUTPUT ?= /tmp/tpcc_lmdb.svg
+TPCC_SQLITE_PROFILE ?= balanced
 
-.PHONY: test test-python test-wasm test-slt test-all wasm-build check tpcc tpcc-dual cargo-check build wasm-examples native-examples fmt clippy
+.PHONY: test test-python test-wasm test-slt test-all wasm-build check tpcc tpcc-kitesql-rocksdb tpcc-kitesql-lmdb tpcc-lmdb-flamegraph tpcc-sqlite tpcc-sqlite-practical tpcc-sqlite-balanced tpcc-dual cargo-check build wasm-examples native-examples fmt clippy
 
 ## Run default Rust tests in the current environment (non-WASM).
 test:
@@ -48,9 +52,31 @@ clippy:
 ## Run formatting (check mode) and clippy linting together.
 check: fmt clippy
 
-## Execute the TPCC workload example as a standalone command.
-tpcc:
-	$(CARGO) run -p tpcc --release
+tpcc: tpcc-kitesql-lmdb
+
+## Execute the TPCC workload on KiteSQL with RocksDB storage.
+tpcc-kitesql-rocksdb:
+	$(CARGO) run -p tpcc --release -- --backend kitesql-rocksdb
+
+## Execute the TPCC workload on KiteSQL with LMDB storage.
+tpcc-kitesql-lmdb:
+	$(CARGO) run -p tpcc --release -- --backend kitesql-lmdb
+
+## Execute TPCC on LMDB and emit a pprof flamegraph SVG.
+tpcc-lmdb-flamegraph:
+	CARGO_PROFILE_RELEASE_DEBUG=true $(CARGO) run -p tpcc --release --features pprof -- --backend kitesql-lmdb --measure-time $(TPCC_MEASURE_TIME) --num-ware $(TPCC_NUM_WARE) --pprof-output $(TPCC_PPROF_OUTPUT)
+
+## Execute the TPCC workload on SQLite with the practical profile.
+tpcc-sqlite:
+	$(CARGO) run -p tpcc --release -- --backend sqlite --sqlite-profile $(TPCC_SQLITE_PROFILE) --path kite_sql_tpcc.sqlite
+
+## Execute the TPCC workload on SQLite with the practical profile.
+tpcc-sqlite-practical:
+	$(MAKE) tpcc-sqlite TPCC_SQLITE_PROFILE=practical
+
+## Execute the TPCC workload on SQLite with the balanced profile.
+tpcc-sqlite-balanced:
+	$(MAKE) tpcc-sqlite TPCC_SQLITE_PROFILE=balanced
 
 ## Execute TPCC while mirroring every statement to an in-memory SQLite instance for validation.
 tpcc-dual:

Makefile

@@ -28,7 +28,7 @@
 ## Introduction
 **KiteSQL** is a lightweight embedded relational database for Rust, inspired by **MyRocks** and **SQLite** and fully written in Rust. It is designed to work not only as a SQL engine, but also as a Rust-native data API that can be embedded directly into applications without relying on external services or heavyweight infrastructure.
 
-KiteSQL supports direct SQL execution, typed ORM models, schema migration, and builder-style queries, so you can combine relational power with an API surface that feels natural in Rust.
+KiteSQL supports direct SQL execution, typed ORM models, schema migration, and builder-style queries, so you can combine relational power with an API surface that feels natural in Rust. On native targets, KiteSQL ships with both RocksDB-backed and LMDB-backed persistent storage builders, plus an in-memory builder for tests and temporary workloads.
 
 ## Key Features
 - A lightweight embedded SQL database fully rewritten in Rust
@@ -79,7 +79,8 @@ struct UserSummary {
 }
 
 fn main() -> Result<(), DatabaseError> {
-    let database = DataBaseBuilder::path("./data").build()?;
+    let database = DataBaseBuilder::path("./data").build_rocksdb()?;
+    // Or: let database = DataBaseBuilder::path("./data").build_lmdb()?;
 
     database.migrate::<User>()?;
 
@@ -128,8 +129,20 @@ fn main() -> Result<(), DatabaseError> {
 }
 ```
 
+## Storage Backends
+- `build_rocksdb()` opens a persistent RocksDB-backed database.
+- `build_lmdb()` opens a persistent LMDB-backed database.
+- `build_in_memory()` opens an in-memory database for tests, examples, and temporary workloads.
+- `build_optimistic()` is available on native targets when you specifically want optimistic transactions on top of RocksDB.
+- Cargo features:
+  - `rocksdb` is enabled by default
+  - `lmdb` is optional
+  - `cargo check --no-default-features --features lmdb` builds an LMDB-only native configuration
+
+On native targets, `LMDB` shines when reads dominate, while `RocksDB` is usually the stronger choice when writes do.
+
 👉**more examples**
-- [hello_word](examples/hello_world.rs)
+- [hello_world](examples/hello_world.rs)
 - [transaction](examples/transaction.rs)
 
 
@@ -149,7 +162,7 @@ console.log(rows.map((r) => r.values.map((v) => v.Int32 ?? v)));
 
 ## Python (PyO3)
 - Enable bindings with Cargo feature `python`.
-- Constructor is explicit: `Database(path)`; in-memory usage is `Database.in_memory()`.
+- Constructor is explicit: `Database(path, backend="rocksdb")`; use `backend="lmdb"` to open LMDB. In-memory usage is `Database.in_memory()`.
 - Minimal usage:
 ```python
 import kite_sql
@@ -162,25 +175,24 @@ for row in db.run("select * from demo"):
 ```
 
 ## TPC-C
-Run `make tpcc` (or `cargo run -p tpcc --release`) to execute the benchmark against the default KiteSQL storage.  
+Run `make tpcc` (or `cargo run -p tpcc --release`) to execute the benchmark against the default KiteSQL storage. Use `--backend rocksdb` or `--backend lmdb` to compare the two persistent backends directly.  
 Run `make tpcc-dual` to mirror every TPCC statement to an in-memory SQLite database alongside KiteSQL and assert the two engines return identical results; this target runs for 60 seconds (`--measure-time 60`). Use `cargo run -p tpcc --release -- --backend dual --measure-time <secs>` for a custom duration.
 
 - i9-13900HX
 - 32.0 GB
 - KIOXIA-EXCERIA PLUS G3 SSD
 - Tips: TPC-C currently only supports single thread
 
-All cases have been fully optimized.
-```shell
-<90th Percentile RT (MaxRT)>
-   New-Order : 0.002  (0.005)
-     Payment : 0.001  (0.013)
-Order-Status : 0.002  (0.006)
-    Delivery : 0.010  (0.023)
- Stock-Level : 0.002  (0.017)
-<TpmC>
-27226 Tpmc
-```
+Recent 720-second local comparison on the machine above:
+
+| Backend | TpmC | New-Order p90 | Payment p90 | Order-Status p90 | Delivery p90 | Stock-Level p90 |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: |
+| KiteSQL LMDB | 53510 | 0.001s | 0.001s | 0.001s | 0.002s | 0.001s |
+| KiteSQL RocksDB | 32248 | 0.001s | 0.001s | 0.002s | 0.011s | 0.003s |
+| SQLite balanced | 36273 | 0.001s | 0.001s | 0.001s | 0.001s | 0.001s |
+| SQLite practical | 35516 | 0.001s | 0.001s | 0.001s | 0.001s | 0.001s |
+
+The detailed raw outputs for both runs are recorded in [tpcc/README.md](tpcc/README.md).
 #### 👉[check more](tpcc/README.md)
 
 ## Roadmap

README.md

@@ -14,7 +14,7 @@
 
 use criterion::{criterion_group, criterion_main, Criterion};
 use indicatif::{ProgressBar, ProgressStyle};
-use kite_sql::db::{DataBaseBuilder, ResultIter};
+use kite_sql::db::DataBaseBuilder;
 use kite_sql::errors::DatabaseError;
 #[cfg(unix)]
 use pprof::criterion::{Output, PProfProfiler};
@@ -38,7 +38,7 @@ fn query_cases() -> Vec<(&'static str, &'static str)> {
 }
 
 fn init_kitesql_query_bench() -> Result<(), DatabaseError> {
-    let database = DataBaseBuilder::path(QUERY_BENCH_KITE_SQL_PATH).build()?;
+    let database = DataBaseBuilder::path(QUERY_BENCH_KITE_SQL_PATH).build_rocksdb()?;
     database
         .run("create table t1 (c1 int primary key, c2 int)")?
         .done()?;
@@ -104,7 +104,7 @@ fn query_on_execute(c: &mut Criterion) {
         init_kitesql_query_bench().unwrap();
     }
     let database = DataBaseBuilder::path(QUERY_BENCH_KITE_SQL_PATH)
-        .build()
+        .build_rocksdb()
         .unwrap();
     println!("Table initialization completed");