Skip to content

Commit 55e9ddf

Browse files
KKouldKKould
committed
feat(cargo): gate native backends behind rocksdb/lmdb features
1 parent e2c4e12 commit 55e9ddf

9 files changed

Lines changed: 95 additions & 38 deletions

File tree

‎Cargo.toml‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,19 @@ default-run = "kite_sql"
1717
[[bin]]
1818
name = "kite_sql"
1919
path = "src/bin/server.rs"
20-
required-features = ["net"]
20+
required-features = ["net", "rocksdb"]
2121

2222
[lib]
2323
doctest = false
2424
crate-type = ["cdylib", "rlib"]
2525

2626
[features]
27-
default = ["macros"]
27+
default = ["macros", "rocksdb"]
2828
macros = []
2929
orm = []
30-
net = ["dep:pgwire", "dep:async-trait", "dep:clap", "dep:env_logger", "dep:futures", "dep:log", "dep:tokio"]
30+
rocksdb = ["dep:rocksdb"]
31+
lmdb = ["dep:lmdb", "dep:lmdb-sys"]
32+
net = ["rocksdb", "dep:pgwire", "dep:async-trait", "dep:clap", "dep:env_logger", "dep:futures", "dep:log", "dep:tokio"]
3133
pprof = ["pprof/criterion", "pprof/flamegraph"]
3234
python = ["dep:pyo3"]
3335

@@ -83,9 +85,9 @@ tempfile = { version = "3.10" }
8385
sqlite = { version = "0.34" }
8486

8587
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
86-
rocksdb = { version = "0.23" }
87-
lmdb = { version = "0.8.0" }
88-
lmdb-sys = { version = "0.8.0" }
88+
rocksdb = { version = "0.23", optional = true }
89+
lmdb = { version = "0.8.0", optional = true }
90+
lmdb-sys = { version = "0.8.0", optional = true }
8991

9092
[target.'cfg(target_arch = "wasm32")'.dependencies]
9193
wasm-bindgen = { version = "0.2.106" }

‎README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,10 @@ fn main() -> Result<(), DatabaseError> {
134134
- `build_lmdb()` opens a persistent LMDB-backed database.
135135
- `build_in_memory()` opens an in-memory database for tests, examples, and temporary workloads.
136136
- `build_optimistic()` is available on native targets when you specifically want optimistic transactions on top of RocksDB.
137+
- Cargo features:
138+
- `rocksdb` is enabled by default
139+
- `lmdb` is optional
140+
- `cargo check --no-default-features --features lmdb` builds an LMDB-only native configuration
137141

138142
On native targets, `LMDB` shines when reads dominate, while `RocksDB` is usually the stronger choice when writes do.
139143

‎examples/hello_world.rs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,10 +90,20 @@ mod app {
9090
let backend = env::var("KITESQL_BACKEND").unwrap_or_else(|_| "rocksdb".to_string());
9191

9292
match backend.to_ascii_lowercase().as_str() {
93+
#[cfg(feature = "rocksdb")]
9394
"rocksdb" => run_with_database(DataBaseBuilder::path(EXAMPLE_DB_PATH).build_rocksdb()?),
95+
#[cfg(feature = "lmdb")]
9496
"lmdb" => run_with_database(DataBaseBuilder::path(EXAMPLE_DB_PATH).build_lmdb()?),
9597
other => Err(DatabaseError::InvalidValue(format!(
96-
"unsupported example backend '{other}', expected 'rocksdb' or 'lmdb'"
98+
"unsupported example backend '{other}', expected {}",
99+
{
100+
let mut expected = Vec::new();
101+
#[cfg(feature = "rocksdb")]
102+
expected.push("rocksdb");
103+
#[cfg(feature = "lmdb")]
104+
expected.push("lmdb");
105+
expected.join(" or ")
106+
}
97107
))),
98108
}
99109
}

‎src/db.rs‎

Lines changed: 27 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,10 @@ use crate::optimizer::rule::normalization::NormalizationRuleImpl;
3232
use crate::parser::parse_sql;
3333
use crate::planner::operator::Operator;
3434
use crate::planner::LogicalPlan;
35-
#[cfg(not(target_arch = "wasm32"))]
35+
#[cfg(all(not(target_arch = "wasm32"), feature = "lmdb"))]
3636
use crate::storage::lmdb::{LmdbConfig, LmdbStorage};
3737
use crate::storage::memory::MemoryStorage;
38-
#[cfg(not(target_arch = "wasm32"))]
38+
#[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))]
3939
use crate::storage::rocksdb::{OptimisticRocksStorage, RocksStorage, StorageConfig};
4040
use crate::storage::{StatisticsMetaCache, Storage, TableCache, Transaction, ViewCache};
4141
use crate::types::tuple::{SchemaRef, Tuple};
@@ -117,9 +117,9 @@ pub struct DataBaseBuilder {
117117
scala_functions: ScalaFunctions,
118118
table_functions: TableFunctions,
119119
histogram_buckets: Option<usize>,
120-
#[cfg(not(target_arch = "wasm32"))]
120+
#[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))]
121121
storage_config: StorageConfig,
122-
#[cfg(not(target_arch = "wasm32"))]
122+
#[cfg(all(not(target_arch = "wasm32"), feature = "lmdb"))]
123123
lmdb_config: LmdbConfig,
124124
}
125125

@@ -134,9 +134,9 @@ impl DataBaseBuilder {
134134
scala_functions: Default::default(),
135135
table_functions: Default::default(),
136136
histogram_buckets: None,
137-
#[cfg(not(target_arch = "wasm32"))]
137+
#[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))]
138138
storage_config: Default::default(),
139-
#[cfg(not(target_arch = "wasm32"))]
139+
#[cfg(all(not(target_arch = "wasm32"), feature = "lmdb"))]
140140
lmdb_config: Default::default(),
141141
};
142142
builder = builder.register_scala_function(CharLength::new("char_length".to_lowercase()));
@@ -174,29 +174,38 @@ impl DataBaseBuilder {
174174
}
175175

176176
/// Enables or disables RocksDB statistics collection.
177-
#[cfg(not(target_arch = "wasm32"))]
177+
#[cfg(all(
178+
not(target_arch = "wasm32"),
179+
any(feature = "rocksdb", feature = "lmdb")
180+
))]
178181
pub fn storage_statistics(mut self, enable: bool) -> Self {
179-
self.storage_config.enable_statistics = enable;
180-
self.lmdb_config.enable_statistics = enable;
182+
#[cfg(feature = "rocksdb")]
183+
{
184+
self.storage_config.enable_statistics = enable;
185+
}
186+
#[cfg(feature = "lmdb")]
187+
{
188+
self.lmdb_config.enable_statistics = enable;
189+
}
181190
self
182191
}
183192

184193
/// Sets the LMDB map size in bytes.
185-
#[cfg(not(target_arch = "wasm32"))]
194+
#[cfg(all(not(target_arch = "wasm32"), feature = "lmdb"))]
186195
pub fn lmdb_map_size(mut self, map_size: usize) -> Self {
187196
self.lmdb_config.map_size = map_size;
188197
self
189198
}
190199

191200
/// Sets the LMDB environment flags.
192-
#[cfg(not(target_arch = "wasm32"))]
201+
#[cfg(all(not(target_arch = "wasm32"), feature = "lmdb"))]
193202
pub fn lmdb_flags(mut self, flags: lmdb::EnvironmentFlags) -> Self {
194203
self.lmdb_config.flags = flags;
195204
self
196205
}
197206

198207
/// Enables or disables LMDB `NO_SYNC`.
199-
#[cfg(not(target_arch = "wasm32"))]
208+
#[cfg(all(not(target_arch = "wasm32"), feature = "lmdb"))]
200209
pub fn lmdb_no_sync(mut self, enable: bool) -> Self {
201210
self.lmdb_config
202211
.flags
@@ -205,14 +214,14 @@ impl DataBaseBuilder {
205214
}
206215

207216
/// Sets the maximum number of LMDB readers.
208-
#[cfg(not(target_arch = "wasm32"))]
217+
#[cfg(all(not(target_arch = "wasm32"), feature = "lmdb"))]
209218
pub fn lmdb_max_readers(mut self, max_readers: u32) -> Self {
210219
self.lmdb_config.max_readers = Some(max_readers);
211220
self
212221
}
213222

214223
/// Sets the maximum number of LMDB named databases.
215-
#[cfg(not(target_arch = "wasm32"))]
224+
#[cfg(all(not(target_arch = "wasm32"), feature = "lmdb"))]
216225
pub fn lmdb_max_dbs(mut self, max_dbs: u32) -> Self {
217226
self.lmdb_config.max_dbs = Some(max_dbs);
218227
self
@@ -242,7 +251,7 @@ impl DataBaseBuilder {
242251
}
243252

244253
/// Builds a RocksDB-backed database.
245-
#[cfg(not(target_arch = "wasm32"))]
254+
#[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))]
246255
pub fn build_rocksdb(self) -> Result<Database<RocksStorage>, DatabaseError> {
247256
let storage = RocksStorage::with_config(self.path, self.storage_config)?;
248257

@@ -269,7 +278,7 @@ impl DataBaseBuilder {
269278
}
270279

271280
/// Builds a LMDB-backed database.
272-
#[cfg(not(target_arch = "wasm32"))]
281+
#[cfg(all(not(target_arch = "wasm32"), feature = "lmdb"))]
273282
pub fn build_lmdb(self) -> Result<Database<LmdbStorage>, DatabaseError> {
274283
let storage = LmdbStorage::with_config(self.path, self.lmdb_config)?;
275284

@@ -281,9 +290,9 @@ impl DataBaseBuilder {
281290
)
282291
}
283292

284-
#[cfg(not(target_arch = "wasm32"))]
293+
#[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))]
285294
/// Builds a RocksDB-backed database that uses optimistic transactions.
286-
#[cfg(not(target_arch = "wasm32"))]
295+
#[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))]
287296
pub fn build_optimistic(self) -> Result<Database<OptimisticRocksStorage>, DatabaseError> {
288297
let storage = OptimisticRocksStorage::with_config(self.path, self.storage_config)?;
289298

‎src/errors.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ pub enum DatabaseError {
211211
PrimaryKeyNotFound,
212212
#[error("primaryKey only allows single or multiple values")]
213213
PrimaryKeyTooManyLayers,
214-
#[cfg(not(target_arch = "wasm32"))]
214+
#[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))]
215215
#[error("rocksdb: {0}")]
216216
RocksDB(
217217
#[source]

‎src/lib.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@
1717
//! making it possible to call SQL just like calling a function.
1818
//! It supports most of the syntax of SQL 2016.
1919
//!
20+
//! Native storage backends are feature-gated:
21+
//! - `rocksdb` is enabled by default
22+
//! - `lmdb` is optional
23+
//!
2024
//! KiteSQL provides thread-safe API: [`DataBase::run`](db::Database::run) for running SQL
2125
//!
2226
//! KiteSQL uses [`DataBaseBuilder`](db::DataBaseBuilder) for instance construction,

‎src/python.rs‎

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,10 @@
1616

1717
use crate::db::{DataBaseBuilder, Database, DatabaseIter};
1818
use crate::errors::DatabaseError;
19+
#[cfg(feature = "lmdb")]
1920
use crate::storage::lmdb::LmdbStorage;
2021
use crate::storage::memory::MemoryStorage;
22+
#[cfg(feature = "rocksdb")]
2123
use crate::storage::rocksdb::RocksStorage;
2224
use crate::types::tuple::{SchemaRef, Tuple};
2325
use crate::types::value::DataValue;
@@ -94,14 +96,17 @@ fn schema_to_python(py: Python<'_>, schema: &SchemaRef) -> PyResult<Vec<PyObject
9496
}
9597

9698
enum PythonDatabaseInner {
99+
#[cfg(feature = "lmdb")]
97100
Lmdb(Database<LmdbStorage>),
98101
Memory(Database<MemoryStorage>),
102+
#[cfg(feature = "rocksdb")]
99103
Rocks(Database<RocksStorage>),
100104
}
101105

102106
impl PythonDatabaseInner {
103107
fn run(&self, sql: &str) -> Result<PythonResultIterInner, DatabaseError> {
104108
match self {
109+
#[cfg(feature = "lmdb")]
105110
PythonDatabaseInner::Lmdb(db) => {
106111
let iter = db.run(sql)?;
107112
// DatabaseIter owns state internally; only the type carries the lifetime.
@@ -116,6 +121,7 @@ impl PythonDatabaseInner {
116121
unsafe { std::mem::transmute(iter) };
117122
Ok(PythonResultIterInner::Memory(iter_static))
118123
}
124+
#[cfg(feature = "rocksdb")]
119125
PythonDatabaseInner::Rocks(db) => {
120126
let iter = db.run(sql)?;
121127
// DatabaseIter owns state internally; only the type carries the lifetime.
@@ -128,32 +134,40 @@ impl PythonDatabaseInner {
128134
}
129135

130136
enum PythonResultIterInner {
137+
#[cfg(feature = "lmdb")]
131138
Lmdb(DatabaseIter<'static, LmdbStorage>),
132139
Memory(DatabaseIter<'static, MemoryStorage>),
140+
#[cfg(feature = "rocksdb")]
133141
Rocks(DatabaseIter<'static, RocksStorage>),
134142
}
135143

136144
impl PythonResultIterInner {
137145
fn next_tuple(&mut self) -> Result<Option<&Tuple>, DatabaseError> {
138146
match self {
147+
#[cfg(feature = "lmdb")]
139148
PythonResultIterInner::Lmdb(iter) => iter.next_borrowed_tuple(),
140149
PythonResultIterInner::Memory(iter) => iter.next_borrowed_tuple(),
150+
#[cfg(feature = "rocksdb")]
141151
PythonResultIterInner::Rocks(iter) => iter.next_borrowed_tuple(),
142152
}
143153
}
144154

145155
fn schema(&self) -> &SchemaRef {
146156
match self {
157+
#[cfg(feature = "lmdb")]
147158
PythonResultIterInner::Lmdb(iter) => iter.schema(),
148159
PythonResultIterInner::Memory(iter) => iter.schema(),
160+
#[cfg(feature = "rocksdb")]
149161
PythonResultIterInner::Rocks(iter) => iter.schema(),
150162
}
151163
}
152164

153165
fn done(self) -> Result<(), DatabaseError> {
154166
match self {
167+
#[cfg(feature = "lmdb")]
155168
PythonResultIterInner::Lmdb(iter) => iter.done(),
156169
PythonResultIterInner::Memory(iter) => iter.done(),
170+
#[cfg(feature = "rocksdb")]
157171
PythonResultIterInner::Rocks(iter) => iter.done(),
158172
}
159173
}
@@ -171,19 +185,27 @@ impl PythonDatabase {
171185
pub fn new(path: String, backend: Option<&str>) -> PyResult<Self> {
172186
let backend = backend.unwrap_or("rocksdb").to_ascii_lowercase();
173187
let inner = match backend.as_str() {
188+
#[cfg(feature = "rocksdb")]
174189
"rocksdb" => PythonDatabaseInner::Rocks(
175190
DataBaseBuilder::path(path)
176191
.build_rocksdb()
177192
.map_err(to_py_err)?,
178193
),
194+
#[cfg(feature = "lmdb")]
179195
"lmdb" => PythonDatabaseInner::Lmdb(
180196
DataBaseBuilder::path(path)
181197
.build_lmdb()
182198
.map_err(to_py_err)?,
183199
),
184200
other => {
201+
let mut expected = Vec::new();
202+
#[cfg(feature = "rocksdb")]
203+
expected.push("rocksdb");
204+
#[cfg(feature = "lmdb")]
205+
expected.push("lmdb");
185206
return Err(PyValueError::new_err(format!(
186-
"unsupported backend '{other}', expected 'rocksdb' or 'lmdb'"
207+
"unsupported backend '{other}', expected {}",
208+
expected.join(" or ")
187209
)));
188210
}
189211
};
@@ -325,16 +347,22 @@ mod tests {
325347
) -> PyResult<()> {
326348
run_script(py, module, script, "memory", "")?;
327349

328-
let temp_dir =
329-
TempDir::new().map_err(|e| PyRuntimeError::new_err(format!("create tempdir: {e}")))?;
330-
let path = temp_dir.path().to_string_lossy().to_string();
350+
#[cfg(feature = "rocksdb")]
351+
{
352+
let temp_dir = TempDir::new()
353+
.map_err(|e| PyRuntimeError::new_err(format!("create tempdir: {e}")))?;
354+
let path = temp_dir.path().to_string_lossy().to_string();
331355

332-
run_script(py, module, script, "rocksdb", &path)?;
356+
run_script(py, module, script, "rocksdb", &path)?;
357+
}
333358

334-
let temp_dir =
335-
TempDir::new().map_err(|e| PyRuntimeError::new_err(format!("create tempdir: {e}")))?;
336-
let path = temp_dir.path().to_string_lossy().to_string();
337-
run_script(py, module, script, "lmdb", &path)?;
359+
#[cfg(feature = "lmdb")]
360+
{
361+
let temp_dir = TempDir::new()
362+
.map_err(|e| PyRuntimeError::new_err(format!("create tempdir: {e}")))?;
363+
let path = temp_dir.path().to_string_lossy().to_string();
364+
run_script(py, module, script, "lmdb", &path)?;
365+
}
338366

339367
Ok(())
340368
}

‎src/storage/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,10 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15-
#[cfg(not(target_arch = "wasm32"))]
15+
#[cfg(all(not(target_arch = "wasm32"), feature = "lmdb"))]
1616
pub mod lmdb;
1717
pub mod memory;
18-
#[cfg(not(target_arch = "wasm32"))]
18+
#[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))]
1919
pub mod rocksdb;
2020
pub(crate) mod table_codec;
2121

‎tpcc/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ pprof = ["dep:pprof"]
99
[dependencies]
1010
clap = { version = "4", features = ["derive"] }
1111
chrono = { version = "0.4" }
12-
kite_sql = { path = "..", package = "kite_sql" }
12+
kite_sql = { path = "..", package = "kite_sql", features = ["rocksdb", "lmdb"] }
1313
indicatif = { version = "0.17" }
1414
ordered-float = { version = "4" }
1515
rand = { version = "0.8" }

0 commit comments

Comments
 (0)