refactor(bench): isolate external engine comparisons (#8)

* refactor(bench): isolate external engine comparisons

* ci(test): check dependency graph under all-features and switch to resolver v3

Cargo's resolver 2 unifies features across the default and all-features
builds, so the CI leak check could miss external engines pulled in only
under --all-features. Bumping to resolver 3 keeps builds isolated per
feature set, and the workflow now runs the cargo-tree leak check against
both the default and --all-features dependency graphs.

---------

Co-authored-by: Farhan Syah <bizimpulse@gmail.com>

Author: presempathy-awb

.github/workflows/test.yml

@@ -46,26 +46,31 @@ jobs:
         with:
           components: rustfmt, clippy
 
-      # The comparison benchmark links rocksdb (librocksdb-sys), whose build
-      # runs bindgen and needs libclang. clippy --all-targets compiles it.
-      - name: Install LLVM/Clang
-        run: sudo apt-get update && sudo apt-get install -y clang libclang-dev
-
       - uses: Swatinem/rust-cache@v2
         with:
           prefix-key: lint
 
       - name: Check formatting
         run: cargo fmt --all --check
 
+      - name: Keep external engines out of the PageDB package
+        run: |
+          for features in "" "--all-features"; do
+            if cargo tree -p pagedb $features --edges normal,build,dev --prefix none |
+              rg -q '^(librocksdb-sys|libsqlite3-sys|redb|rocksdb|rusqlite) v'; then
+              echo "::error::External comparison engine leaked into the pagedb dependency graph (${features:-default features})."
+              exit 1
+            fi
+          done
+
       - name: Clippy (all targets, all features)
-        run: cargo clippy --all-targets --all-features -- -D warnings
+        run: cargo clippy -p pagedb --all-targets --all-features -- -D warnings
 
       - name: Build docs
-        run: cargo doc --no-deps --all-features
+        run: cargo doc -p pagedb --no-deps --all-features
 
       - name: Run doctests
-        run: cargo test --doc --all-features
+        run: cargo test -p pagedb --doc --all-features
 
       # Enforces the rule in CLAUDE.md: no checklist-letter or build-order
       # scaffolding may leak into source / Cargo.toml / README. Greps for
@@ -95,19 +100,6 @@ jobs:
       - name: Install Rust
         uses: dtolnay/rust-toolchain@stable
 
-      # nextest compiles the dev-dependency graph (incl. rocksdb →
-      # librocksdb-sys → bindgen → libclang). Windows runner images ship LLVM;
-      # Linux and macOS need it installed.
-      - name: Install LLVM/Clang (Linux)
-        if: runner.os == 'Linux'
-        run: sudo apt-get update && sudo apt-get install -y clang libclang-dev
-
-      - name: Install LLVM/Clang (macOS)
-        if: runner.os == 'macOS'
-        run: |
-          brew install llvm
-          echo "LIBCLANG_PATH=$(brew --prefix llvm)/lib" >> "$GITHUB_ENV"
-
       - uses: Swatinem/rust-cache@v2
         with:
           prefix-key: test-${{ matrix.os }}
@@ -120,7 +112,7 @@ jobs:
       # Always nextest — crash-recovery tests rely on the serial group in
       # .config/nextest.toml that `cargo test` ignores.
       - name: Run tests
-        run: cargo nextest run --all-features --no-fail-fast
+        run: cargo nextest run -p pagedb --all-features --no-fail-fast
 
       - name: Upload nextest report
         if: always()
@@ -167,7 +159,7 @@ jobs:
           prefix-key: cross-${{ matrix.target }}
 
       - name: cargo check --lib
-        run: cargo check --target ${{ matrix.target }} --lib
+        run: cargo check -p pagedb --target ${{ matrix.target }} --lib
 
   # ─────────────────────────────────────────────────────────────────────────
   wasm:
@@ -196,7 +188,7 @@ jobs:
           prefix-key: wasm-${{ matrix.target }}
 
       - name: cargo check --lib
-        run: cargo check --target ${{ matrix.target }} --lib ${{ matrix.features }}
+        run: cargo check -p pagedb --target ${{ matrix.target }} --lib ${{ matrix.features }}
 
   # ─────────────────────────────────────────────────────────────────────────
   features:
@@ -216,17 +208,12 @@ jobs:
       - name: Install Rust
         uses: dtolnay/rust-toolchain@stable
 
-      # --benches compiles the comparison bench, which links rocksdb
-      # (librocksdb-sys) — its build runs bindgen and needs libclang.
-      - name: Install LLVM/Clang
-        run: sudo apt-get update && sudo apt-get install -y clang libclang-dev
-
       - uses: Swatinem/rust-cache@v2
         with:
           prefix-key: features
 
       - name: cargo check
-        run: cargo check --lib --bins --tests --benches ${{ matrix.flags }}
+        run: cargo check -p pagedb --lib --bins --tests --benches ${{ matrix.flags }}
 
   # ─────────────────────────────────────────────────────────────────────────
   bench:
@@ -238,8 +225,8 @@ jobs:
       - name: Install Rust
         uses: dtolnay/rust-toolchain@stable
 
-      # cargo bench builds the dev-dependency graph (rocksdb → librocksdb-sys →
-      # bindgen → libclang), even when only the btree/segment benches are named.
+      # The isolated comparison package links RocksDB (librocksdb-sys), whose
+      # build runs bindgen and needs libclang.
       - name: Install LLVM/Clang
         run: sudo apt-get update && sudo apt-get install -y clang libclang-dev
 
@@ -251,7 +238,9 @@ jobs:
       # dispatched workflow so we don't pay the runtime on every PR. This
       # job catches "the bench no longer compiles" regressions.
       - name: Build benches
-        run: cargo bench --no-run --bench btree --bench segment
+        run: |
+          cargo bench --no-run -p pagedb --bench segment
+          cargo bench --no-run -p pagedb-engine-comparison --bench btree --bench comparison
 
   # ─────────────────────────────────────────────────────────────────────────
   audit:

Cargo.lock

@@ -1,3 +1,8 @@
+[workspace]
+members = ["benchmarks/engine-comparison"]
+default-members = ["."]
+resolver = "3"
+
 [package]
 name = "pagedb"
 version = "0.1.0"
@@ -123,37 +128,22 @@ tokio = { version = "1", features = [
     "test-util",
 ] }
 tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
-redb = "2"
 tempfile = "3"
 aes-gcm = "0.10"
 fluxbench = "0.1"
-rocksdb = { version = "0.24", default-features = false, features = ["lz4"] }
-rusqlite = { version = "0.40", features = ["bundled"] }
-fastrand = "2"
 
 [[bin]]
 name = "pagedb-fsck"
 path = "src/bin/pagedb-fsck.rs"
 
-# `test = false`: these are benchmarks, not tests. Without it, `cargo test`
-# (and nextest) compile every `harness = false` bench and run its `main` as a
-# test — pulling heavy dev-deps (rocksdb → libclang) into the test build on
-# every platform and executing benchmarks during the test run.
-[[bench]]
-name = "btree"
-harness = false
-test = false
-
+# These are executable benchmarks, not test harnesses. The cross-engine suite
+# is a separate non-default workspace package so normal PageDB tests never
+# resolve its RocksDB, redb, or SQLite dependencies.
 [[bench]]
 name = "segment"
 harness = false
 test = false
 
-[[bench]]
-name = "comparison"
-harness = false
-test = false
-
 [profile.release]
 lto = "thin"
 codegen-units = 1

Cargo.toml

@@ -57,14 +57,20 @@ pagedb's answer: **one substrate, two surfaces.** Engines that want their own fo
 
 ## Benchmarks
 
-Measured on native NVMe, AES-NI host, single thread, via [`fluxbench`](./benches). Reproduce with:
+Measured on native NVMe, AES-NI host, single thread, via `fluxbench`. Reproduce the
+PageDB-owned benches and the isolated cross-engine suite with:
 
 ```bash
-cargo bench --bench btree
-cargo bench --bench comparison
 cargo bench --bench segment
+cargo bench -p pagedb-engine-comparison --bench btree
+cargo bench -p pagedb-engine-comparison --bench comparison
 ```
 
+The comparison suite is a non-default workspace package under
+[`benchmarks/engine-comparison`](./benchmarks/engine-comparison). Normal
+`cargo test` and `cargo nextest` runs for the `pagedb` package therefore do not
+resolve or compile RocksDB, redb, or SQLite.
+
 ### vs. redb (B+ tree, in-process)
 
 | Workload                       |      pagedb |    redb | Speedup vs redb |

README.md

@@ -0,0 +1,30 @@
+[package]
+name = "pagedb-engine-comparison"
+version = "0.0.0"
+edition = "2024"
+rust-version = "1.85"
+publish = false
+license = "MIT OR Apache-2.0"
+description = "Benchmark-only cross-engine comparison harness for PageDB"
+
+[dev-dependencies]
+fastrand = "2"
+fluxbench = "0.1"
+pagedb = { path = "../.." }
+redb = "2"
+rocksdb = { version = "0.24", default-features = false, features = ["lz4"] }
+rusqlite = { version = "0.40", features = ["bundled"] }
+tempfile = "3"
+tokio = { version = "1", features = ["rt", "sync", "time"] }
+
+[[bench]]
+name = "btree"
+path = "btree.rs"
+harness = false
+test = false
+
+[[bench]]
+name = "comparison"
+path = "comparison.rs"
+harness = false
+test = false

benchmarks/engine-comparison/Cargo.toml

@@ -9,7 +9,7 @@
 //! of the threat model; comparing redb vs `pagedb-plain` isolates the
 //! structural cost (CoW shadow paging, AAD, MAC-only, durable reader pins).
 //!
-//! Run with: `cargo bench --bench btree`
+//! Run with: `cargo bench -p pagedb-engine-comparison --bench btree`
 
 #![allow(dead_code)] // verify/synthetic/compare placeholder structs
 

benches/btree.rs benchmarks/engine-comparison/btree.rsbenches/btree.rs renamed to benchmarks/engine-comparison/btree.rs

@@ -10,7 +10,7 @@
 //!   - individual writes / batch writes: per-txn latency, many samples
 //!   - random reads / range reads: per-op latency on a preloaded DB
 //!
-//! Run with: `cargo bench --bench comparison`
+//! Run with: `cargo bench -p pagedb-engine-comparison --bench comparison`
 
 #![allow(dead_code)] // verify/synthetic/compare placeholder structs