Move acceptance scale checks to Criterion benches

Author: lucidprogrammer

benches/acceptance_three_paradigm_query.rs

@@ -0,0 +1,175 @@
+mod common;
+
+use common::scale::{empty, params};
+use contextdb_core::Value;
+use contextdb_engine::Database;
+use criterion::{BatchSize, Criterion, criterion_group, criterion_main};
+use std::time::{Duration, Instant};
+use uuid::Uuid;
+
+fn seed_fixture() -> Database {
+    let db = Database::open_memory();
+    db.execute(
+        "CREATE TABLE observations (id UUID PRIMARY KEY, obs_type TEXT NOT NULL, source TEXT NOT NULL, embedding VECTOR(3))",
+        &empty(),
+    )
+    .unwrap();
+    db.execute(
+        "CREATE TABLE entities (id UUID PRIMARY KEY, entity_type TEXT NOT NULL, name TEXT NOT NULL)",
+        &empty(),
+    )
+    .unwrap();
+    db.execute(
+        "CREATE TABLE decisions (id UUID PRIMARY KEY, description TEXT NOT NULL, status TEXT NOT NULL)",
+        &empty(),
+    )
+    .unwrap();
+
+    let gate = Uuid::from_u128(1);
+    let parking = Uuid::from_u128(2);
+    db.execute(
+        "INSERT INTO entities (id, entity_type, name) VALUES ($id, 'LOCATION', 'main-gate')",
+        &params(vec![("id", Value::Uuid(gate))]),
+    )
+    .unwrap();
+    db.execute(
+        "INSERT INTO entities (id, entity_type, name) VALUES ($id, 'LOCATION', 'parking-lot')",
+        &params(vec![("id", Value::Uuid(parking))]),
+    )
+    .unwrap();
+
+    let dec_gate = Uuid::from_u128(10);
+    let dec_park = Uuid::from_u128(11);
+    db.execute(
+        "INSERT INTO decisions (id, description, status) VALUES ($id, 'Alert on unknown person at gate', 'active')",
+        &params(vec![("id", Value::Uuid(dec_gate))]),
+    )
+    .unwrap();
+    db.execute(
+        "INSERT INTO decisions (id, description, status) VALUES ($id, 'Log vehicle plates in parking', 'superseded')",
+        &params(vec![("id", Value::Uuid(dec_park))]),
+    )
+    .unwrap();
+
+    db.execute(
+        "INSERT INTO GRAPH (source_id, target_id, edge_type) VALUES ($src, $tgt, 'BASED_ON')",
+        &params(vec![
+            ("src", Value::Uuid(dec_gate)),
+            ("tgt", Value::Uuid(gate)),
+        ]),
+    )
+    .unwrap();
+    db.execute(
+        "INSERT INTO GRAPH (source_id, target_id, edge_type) VALUES ($src, $tgt, 'BASED_ON')",
+        &params(vec![
+            ("src", Value::Uuid(dec_park)),
+            ("tgt", Value::Uuid(parking)),
+        ]),
+    )
+    .unwrap();
+
+    let obs_g1 = Uuid::from_u128(20);
+    let obs_g2 = Uuid::from_u128(21);
+    let obs_p1 = Uuid::from_u128(22);
+    db.execute(
+        "INSERT INTO observations (id, obs_type, source, embedding) VALUES ($id, 'person_detected', 'cam-gate', [0.9, 0.1, 0.0])",
+        &params(vec![("id", Value::Uuid(obs_g1))]),
+    )
+    .unwrap();
+    db.execute(
+        "INSERT INTO observations (id, obs_type, source, embedding) VALUES ($id, 'person_detected', 'cam-gate', [0.95, 0.05, 0.0])",
+        &params(vec![("id", Value::Uuid(obs_g2))]),
+    )
+    .unwrap();
+    db.execute(
+        "INSERT INTO observations (id, obs_type, source, embedding) VALUES ($id, 'vehicle_detected', 'cam-parking', [0.0, 0.95, 0.05])",
+        &params(vec![("id", Value::Uuid(obs_p1))]),
+    )
+    .unwrap();
+
+    db.execute(
+        "INSERT INTO GRAPH (source_id, target_id, edge_type) VALUES ($src, $tgt, 'OBSERVED_ON')",
+        &params(vec![
+            ("src", Value::Uuid(obs_g1)),
+            ("tgt", Value::Uuid(gate)),
+        ]),
+    )
+    .unwrap();
+    db.execute(
+        "INSERT INTO GRAPH (source_id, target_id, edge_type) VALUES ($src, $tgt, 'OBSERVED_ON')",
+        &params(vec![
+            ("src", Value::Uuid(obs_g2)),
+            ("tgt", Value::Uuid(gate)),
+        ]),
+    )
+    .unwrap();
+    db.execute(
+        "INSERT INTO GRAPH (source_id, target_id, edge_type) VALUES ($src, $tgt, 'OBSERVED_ON')",
+        &params(vec![
+            ("src", Value::Uuid(obs_p1)),
+            ("tgt", Value::Uuid(parking)),
+        ]),
+    )
+    .unwrap();
+    db
+}
+
+fn run_query_and_assert(db: &Database) {
+    let result = db
+        .execute(
+            "WITH similar_obs AS (\
+            SELECT id FROM observations \
+            ORDER BY embedding <=> $query_vec \
+            LIMIT 5\
+        ), \
+        reached AS (\
+            SELECT b_id FROM GRAPH_TABLE(\
+                edges MATCH (a)-[:OBSERVED_ON]->{1,1}(entity)<-[:BASED_ON]-(b) \
+                WHERE a.id IN (SELECT id FROM similar_obs) \
+                COLUMNS (b.id AS b_id)\
+            )\
+        ) \
+        SELECT d.id, d.description \
+        FROM decisions d \
+        INNER JOIN reached r ON d.id = r.b_id \
+        WHERE d.status = 'active'",
+            &params(vec![("query_vec", Value::Vector(vec![1.0, 0.0, 0.0]))]),
+        )
+        .unwrap();
+    assert_eq!(result.rows.len(), 1);
+    let desc = match &result.rows[0][1] {
+        Value::Text(value) => value,
+        other => panic!("expected Text for description, got: {other:?}"),
+    };
+    assert!(desc.contains("gate"), "expected gate decision, got: {desc}");
+    assert!(
+        !desc.contains("parking"),
+        "superseded parking decision should not appear"
+    );
+}
+
+fn timed_query_and_assert(db: &Database) {
+    let started = Instant::now();
+    run_query_and_assert(db);
+    let elapsed = started.elapsed();
+    assert!(
+        elapsed < Duration::from_millis(50),
+        "query took {}ms, expected < 50ms",
+        elapsed.as_millis()
+    );
+}
+
+fn bench_three_paradigm_recall_under_50ms(c: &mut Criterion) {
+    let db = seed_fixture();
+    run_query_and_assert(&db);
+    c.bench_function("three_paradigm_recall_under_50ms", |b| {
+        b.iter_batched(
+            seed_fixture,
+            |db| timed_query_and_assert(&db),
+            BatchSize::LargeInput,
+        )
+    });
+}
+
+criterion_group!(benches, bench_three_paradigm_recall_under_50ms);
+criterion_main!(benches);

benches/common/mod.rs

@@ -1,5 +1,6 @@
 #![allow(dead_code)]
 
 pub mod process;
+pub mod scale;
 pub mod sync;
 pub mod workloads;

benches/common/scale.rs

@@ -0,0 +1,10 @@
+use contextdb_core::Value;
+use std::collections::HashMap;
+
+pub fn empty() -> HashMap<String, Value> {
+    HashMap::new()
+}
+
+pub fn params(pairs: Vec<(&str, Value)>) -> HashMap<String, Value> {
+    pairs.into_iter().map(|(k, v)| (k.to_string(), v)).collect()
+}

benches/indexed_scan_filter_cg_entity_list.rs

@@ -0,0 +1,100 @@
+mod common;
+
+use common::scale::{empty, params};
+use contextdb_core::Value;
+use contextdb_engine::Database;
+use criterion::{BatchSize, Criterion, criterion_group, criterion_main};
+use std::time::{Duration, Instant};
+use uuid::Uuid;
+
+const ROWS: i64 = 5_000;
+
+fn seed_entities() -> Database {
+    let db = Database::open_memory();
+    db.execute(
+        "CREATE TABLE entities (id UUID PRIMARY KEY, entity_type TEXT, name TEXT, created_at TIMESTAMP)",
+        &empty(),
+    )
+    .unwrap();
+    db.execute(
+        "CREATE INDEX idx_entity_type ON entities (entity_type, created_at DESC, id DESC)",
+        &empty(),
+    )
+    .unwrap();
+    for i in 0..ROWS {
+        let entity_type = if i % 2 == 0 { "Service" } else { "Database" };
+        let name = if i % 5 == 0 {
+            format!("pay-entity-{i}")
+        } else {
+            format!("misc-entity-{i}")
+        };
+        db.execute(
+            "INSERT INTO entities (id, entity_type, name, created_at) VALUES ($id, $et, $n, $ts)",
+            &params(vec![
+                ("id", Value::Uuid(Uuid::new_v4())),
+                ("et", Value::Text(entity_type.into())),
+                ("n", Value::Text(name)),
+                ("ts", Value::Timestamp(1_700_000_000_000 + i)),
+            ]),
+        )
+        .unwrap();
+    }
+    db
+}
+
+fn run_filtered_query(db: &Database, rows_examined_limit: u64) {
+    db.__reset_rows_examined();
+    let result = db
+        .execute(
+            "SELECT id, entity_type, name FROM entities \
+             WHERE entity_type = $et AND name LIKE $pat \
+             ORDER BY created_at DESC, id DESC",
+            &params(vec![
+                ("et", Value::Text("Service".into())),
+                ("pat", Value::Text("%pay%".into())),
+            ]),
+        )
+        .unwrap();
+    assert_eq!(result.rows.len(), 500);
+    assert_eq!(result.trace.physical_plan, "IndexScan");
+    assert!(result.trace.sort_elided);
+    assert!(
+        db.__rows_examined() <= rows_examined_limit,
+        "IndexScan must not examine the full 5K table; got {}",
+        db.__rows_examined()
+    );
+}
+
+fn seed_and_assert() {
+    let db = seed_entities();
+    run_filtered_query(&db, 3000);
+}
+
+fn timed_queries_and_assert(db: &Database) {
+    let mut samples = Vec::new();
+    for _ in 0..20 {
+        let started = Instant::now();
+        run_filtered_query(db, 3000);
+        samples.push(started.elapsed());
+    }
+    samples.sort();
+    let p95 = samples[(samples.len() * 95 / 100).min(samples.len() - 1)];
+    assert!(
+        p95 <= Duration::from_millis(150),
+        "p95 {p95:?} exceeds 150ms budget"
+    );
+}
+
+fn bench_cg02_entity_list_filter_under_budget(c: &mut Criterion) {
+    seed_and_assert();
+    c.bench_function("cg02_entity_list_filter_under_budget", |b| {
+        b.iter_batched(
+            seed_entities,
+            |db| timed_queries_and_assert(&db),
+            BatchSize::LargeInput,
+        )
+    });
+}
+
+criterion_group!(benches, bench_cg02_entity_list_filter_under_budget);
+criterion_main!(benches);

benches/indexed_scan_filter_file_backed_open.rs

@@ -0,0 +1,102 @@
+mod common;
+
+use common::scale::{empty, params};
+use contextdb_core::Value;
+use contextdb_engine::Database;
+use criterion::{Criterion, criterion_group, criterion_main};
+use std::time::{Duration, Instant};
+use tempfile::TempDir;
+use uuid::Uuid;
+
+const ROWS: i64 = 100_000;
+const SAMPLE: i64 = 42_000;
+
+fn insert_rows(path: &std::path::Path) -> Uuid {
+    let db = Database::open(path).unwrap();
+    db.execute(
+        "CREATE TABLE t (id UUID PRIMARY KEY, bucket INTEGER)",
+        &empty(),
+    )
+    .unwrap();
+    db.execute("CREATE INDEX idx_bucket ON t (bucket)", &empty())
+        .unwrap();
+    let mut sampled_id = None;
+    for i in 0..ROWS {
+        if i % 1_000 == 0 {
+            db.execute("BEGIN", &empty()).unwrap();
+        }
+        let id = Uuid::new_v4();
+        if i == SAMPLE {
+            sampled_id = Some(id);
+        }
+        db.execute(
+            "INSERT INTO t (id, bucket) VALUES ($id, $b)",
+            &params(vec![
+                ("id", Value::Uuid(id)),
+                ("b", Value::Int64(i % 10_000)),
+            ]),
+        )
+        .unwrap();
+        if i % 1_000 == 999 {
+            db.execute("COMMIT", &empty()).unwrap();
+        }
+    }
+    sampled_id.unwrap()
+}
+
+fn assert_reopened(path: &std::path::Path, sampled_id: Uuid) {
+    let db = Database::open(path).unwrap();
+    let count = db.execute("SELECT COUNT(*) FROM t", &empty()).unwrap();
+    assert_eq!(count.rows, vec![vec![Value::Int64(ROWS)]]);
+    let r = db
+        .execute("SELECT id FROM t WHERE bucket = 42", &empty())
+        .unwrap();
+    assert_eq!(r.trace.physical_plan, "IndexScan");
+    assert_eq!(r.rows.len(), 10);
+    assert!(db.__rows_examined() <= 50, "got {}", db.__rows_examined());
+    let sampled = db
+        .execute(
+            "SELECT id, bucket FROM t WHERE id = $id",
+            &params(vec![("id", Value::Uuid(sampled_id))]),
+        )
+        .unwrap();
+    assert_eq!(sampled.rows.len(), 1);
+    assert_eq!(sampled.rows[0][0], Value::Uuid(sampled_id));
+    assert_eq!(sampled.rows[0][1], Value::Int64(SAMPLE % 10_000));
+}
+
+fn seed_reopen_and_assert() {
+    let tmp = TempDir::new().unwrap();
+    let path = tmp.path().join("db");
+    let sampled_id = insert_rows(&path);
+    assert_reopened(&path, sampled_id);
+}
+
+fn timed_seed_reopen_and_assert() {
+    let tmp = TempDir::new().unwrap();
+    let path = tmp.path().join("db");
+    let insert_started = Instant::now();
+    let sampled_id = insert_rows(&path);
+    let insert_elapsed = insert_started.elapsed();
+    assert!(
+        insert_elapsed < Duration::from_secs(900),
+        "100K file-backed INSERTs must finish in <900s, took {insert_elapsed:?}"
+    );
+    let reopen_started = Instant::now();
+    assert_reopened(&path, sampled_id);
+    let reopen_elapsed = reopen_started.elapsed();
+    assert!(
+        reopen_elapsed < Duration::from_secs(10),
+        "100K-row reopen must complete in <10s; took {reopen_elapsed:?}"
+    );
+}
+
+fn bench_file_backed_100k_rows_open_upper_bound(c: &mut Criterion) {
+    seed_reopen_and_assert();
+    c.bench_function("file_backed_100k_rows_open_upper_bound", |b| {
+        b.iter(timed_seed_reopen_and_assert)
+    });
+}
+
+criterion_group!(benches, bench_file_backed_100k_rows_open_upper_bound);
+criterion_main!(benches);