feat(eqp): render LIST SUBQUERY + BLOOM FILTER for a non-correlated IN (SELECT) (B9a)

MagicalTux · claude · MagicalTux · commit dd700fc0fb84 · 2026-07-04T05:46:21.000+09:00
A single non-correlated `[NOT] IN (SELECT …)` in the WHERE now renders a
`LIST SUBQUERY 1` node (child: the body's plan, then a `CREATE BLOOM FILTER`
child) after the table access, matching SQLite — graphite previously rendered
just the bare access.

Gated to the provably byte-exact subset so nothing is emitted into a
non-matching plan: graphite's access is a bare `SCAN` (so there is no seek to
diverge from SQLite's cost-model outer choice — an `… AND c=?` that graphite
seeks makes the line a SEARCH and declines here, dodging the cases where SQLite
scans-plus-bloom where graphite would seek), and either the form is `NOT IN`
(never seeks the IN column) or the IN column is not seekable (so SQLite also
scans). This covers the common `NOT IN (SELECT …)`. A positive `IN` on an
indexed / rowid column (SQLite serves a per-candidate SEARCH), and a correlated /
compound / cross-position subquery, keep graphite's prior bare SCAN — the
executor per-value seek is a documented follow-up (B9a-seek).

New free fns collect_in_selects / single_where_in_select. Removed the now-handled
`b IN (SELECT)` case from the scalar-subquery decline test.

Co-Authored-By: Claude Opus 4.8 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/ROADMAP.md b/ROADMAP.md
@@ -1173,11 +1173,21 @@ plan **and** row differential parity vs the pinned `sqlite3 3.50.4`; the executo
 already returns correct rows for every one (B9i differs only in an SQL-unspecified
 tie/representative order), so they are perf/EQP-fidelity work, not correctness:
 
-- **B9a — `IN (SELECT …)` → `LIST SUBQUERY` + bloom filter.** A `WHERE col IN
-  (<non-correlated SELECT>)` seeks the outer table per candidate and materializes the
-  subquery as a `LIST SUBQUERY N` sibling with a `CREATE BLOOM FILTER` node; graphite
-  currently `SCAN`s and folds the `IN` in the tree-walker. Needs the list-materialize
-  + bloom EQP model (and, for the VDBE, the seek-per-value plan).
+- **B9a — `IN (SELECT …)` → `LIST SUBQUERY` + bloom filter.** ◐ **Partly done (EQP).**
+  A single non-correlated `[NOT] IN (SELECT …)` now renders `LIST SUBQUERY 1` (child =
+  the body's plan, then a `CREATE BLOOM FILTER` child) after the access, for the
+  *provably byte-exact* subset: graphite's access is a bare `SCAN` (so no seek to
+  diverge from SQLite's cost-model outer choice — an `… AND c=?` that graphite seeks
+  makes the line a SEARCH and declines, dodging the cases where SQLite scans-plus-bloom
+  where graphite would seek), and either the form is `NOT IN` or the IN column is not
+  seekable. This covers the common `NOT IN (SELECT …)`. *Still open (**B9a-seek**):* a
+  positive `IN` on an **indexed / rowid** column, which SQLite serves with a
+  per-candidate `SEARCH t (col=?)` — graphite folds the `IN` in the tree-walker and
+  scans, so it declines (no wrong node emitted). That needs the executor to evaluate
+  the non-correlated subquery to a value list and seek per value (the `find_in_constraint`
+  + `try_index_in` path already seeks a literal `IN` list), plus the outer-access EQP;
+  and a compound / correlated body (`CORRELATED LIST SUBQUERY`, shared-id bump) stays
+  deferred.
 - **B9b — window-function EQP.** `… OVER (…)` renders `CO-ROUTINE (subquery-N)` over
   the windowed input; the `(subquery-N)` label is codegen-order-fragile (see the
   `schema-sql-canonicalization` note), so this needs a deterministic-numbering model
diff --git a/src/exec/mod.rs b/src/exec/mod.rs
@@ -14358,6 +14358,45 @@ impl Connection {
                 }
             }
         }
+        // A single non-correlated `[NOT] IN (SELECT …)` in the WHERE renders a
+        // `LIST SUBQUERY 1` node (child = the body's plan, then a `CREATE BLOOM FILTER`
+        // sibling under it) after the scan. We render it only where the whole plan is
+        // provably byte-exact: graphite's access is a bare `SCAN {label}` — so there is
+        // no seek to diverge from SQLite's cost-model choice (an `… AND c=?` that
+        // graphite seeks makes the line a SEARCH and declines here, dodging the case
+        // where SQLite scans-plus-bloom where graphite would seek) — AND either the form
+        // is `NOT IN` (which never seeks the IN column) or the IN column is not seekable
+        // (so SQLite also scans; a positive `IN` on an indexed / rowid column, which
+        // SQLite SEARCHes per candidate, declines — that seek is roadmap B9a-seek).
+        if from.joins.is_empty()
+            && single_scan_detail.as_deref() == Some(alloc::format!("SCAN {label}").as_str())
+        {
+            if let Some((body, negated, operand)) = sel
+                .where_clause
+                .as_ref()
+                .and_then(|w| single_where_in_select(w))
+            {
+                let operand_is_rowid = matches!(operand, Expr::Column { column, .. }
+                    if is_rowid_alias(column)
+                        && !meta.columns.iter().any(|c| c.name.eq_ignore_ascii_case(column)));
+                let in_col_seekable = operand_is_rowid
+                    || col_index(operand, &meta.columns).is_some_and(|c| {
+                        meta.ipk == Some(c)
+                            || self
+                                .indexes_of(&from.first.name)
+                                .is_ok_and(|ixs| ixs.iter().any(|i| i.cols.first() == Some(&c)))
+                    });
+                if (negated || !in_col_seekable) && self.eqp_scalar_bodies_renderable(&[body]) {
+                    let list_id = *next_id;
+                    *next_id += 1;
+                    out.push((list_id, parent, String::from("LIST SUBQUERY 1")));
+                    self.eqp_select(body, list_id, next_id, out, params)?;
+                    let bloom_id = *next_id;
+                    *next_id += 1;
+                    out.push((bloom_id, list_id, String::from("CREATE BLOOM FILTER")));
+                }
+            }
+        }
         // Fold each join in FROM order, tracking the accumulated left columns so
         // the rowid-seek decision (shared with the executor via `rowid_join_seek`)
         // can print `SEARCH … USING INTEGER PRIMARY KEY (rowid=?)` in lockstep
@@ -27149,6 +27188,87 @@ fn collect_where_scalar_subqueries<'a>(e: &'a Expr, out: &mut Vec<&'a Select>) -
     }
 }
 
+/// Collect every `[NOT] IN (SELECT …)` in `e` as `(body, negated, operand)`, setting
+/// `other` if any scalar `(SELECT …)` / `EXISTS` is present. Does NOT descend into a
+/// subquery body (that is the body's own plan). Lifetime-preserving (mirrors
+/// [`collect_where_scalar_subqueries`]) so the caller can hold the borrowed refs.
+fn collect_in_selects<'a>(
+    e: &'a Expr,
+    ins: &mut Vec<(&'a Select, bool, &'a Expr)>,
+    other: &mut bool,
+) {
+    match e {
+        Expr::Subquery(_) | Expr::Exists { .. } => *other = true,
+        Expr::InSelect {
+            expr,
+            select,
+            negated,
+        } => {
+            ins.push((select.as_ref(), *negated, expr.as_ref()));
+            collect_in_selects(expr, ins, other);
+        }
+        Expr::Unary { expr, .. }
+        | Expr::IsNull { expr, .. }
+        | Expr::Cast { expr, .. }
+        | Expr::Paren(expr)
+        | Expr::Collate { expr, .. } => collect_in_selects(expr, ins, other),
+        Expr::Binary { left, right, .. } => {
+            collect_in_selects(left, ins, other);
+            collect_in_selects(right, ins, other);
+        }
+        Expr::Function { args, .. } | Expr::RowValue(args) => {
+            for a in args {
+                collect_in_selects(a, ins, other);
+            }
+        }
+        Expr::InList { expr, list, .. } => {
+            collect_in_selects(expr, ins, other);
+            for a in list {
+                collect_in_selects(a, ins, other);
+            }
+        }
+        Expr::Between {
+            expr, low, high, ..
+        } => {
+            collect_in_selects(expr, ins, other);
+            collect_in_selects(low, ins, other);
+            collect_in_selects(high, ins, other);
+        }
+        Expr::Case {
+            operand,
+            when_then,
+            else_result,
+        } => {
+            if let Some(o) = operand {
+                collect_in_selects(o, ins, other);
+            }
+            for (w, t) in when_then {
+                collect_in_selects(w, ins, other);
+                collect_in_selects(t, ins, other);
+            }
+            if let Some(el) = else_result {
+                collect_in_selects(el, ins, other);
+            }
+        }
+        _ => {}
+    }
+}
+
+/// The single `[NOT] IN (SELECT …)` subquery in `e`, as `(body, negated, operand)`,
+/// but only when it is the *sole* subquery anywhere in `e` (any scalar `(SELECT …)`,
+/// `EXISTS`, or a second `IN (SELECT)` returns `None`, so the shared subquery-id
+/// counter stays a clean `1`). Used to render a `LIST SUBQUERY 1` + `CREATE BLOOM
+/// FILTER` node.
+fn single_where_in_select(e: &Expr) -> Option<(&Select, bool, &Expr)> {
+    let mut ins = Vec::new();
+    let mut other = false;
+    collect_in_selects(e, &mut ins, &mut other);
+    if other || ins.len() != 1 {
+        return None;
+    }
+    ins.into_iter().next()
+}
+
 /// Reconstruct the `sql` text stored in `sqlite_schema` for a `CREATE` statement
 /// the way SQLite does (`sqlite3EndTable` / `sqlite3CreateIndex`): a regenerated
 /// `CREATE <TYPE> ` head — which drops `TEMP`/`IF NOT EXISTS` and normalises the
diff --git a/tests/eqp_in_subquery.rs b/tests/eqp_in_subquery.rs
@@ -0,0 +1,107 @@
+//! B9a — a single non-correlated `[NOT] IN (SELECT …)` in the WHERE renders a
+//! `LIST SUBQUERY 1` node (its body plan + a `CREATE BLOOM FILTER` child) after the
+//! table access, matching SQLite. graphite used to render just the bare access.
+//!
+//! Rendered only where the whole plan is provably byte-exact: graphite's access is a
+//! bare `SCAN` (so there is no seek to diverge from SQLite's cost-model choice), and
+//! either the form is `NOT IN` (which never seeks the IN column) or the IN column is
+//! not seekable (so SQLite also scans). A *positive* `IN` on an indexed / rowid column
+//! — which SQLite serves with a per-candidate `SEARCH` — declines (that seek is a
+//! separate follow-up), as do a correlated / compound / cross-position subquery.
+//! Verified vs the sqlite3 3.50.4 CLI.
+
+#![cfg(feature = "std")]
+
+use std::process::Command;
+
+fn sqlite3_available() -> bool {
+    Command::new("sqlite3").arg("--version").output().is_ok()
+}
+
+fn plan(bin: &str, base: &str, sql: &str) -> String {
+    let full = format!("{base} EXPLAIN QUERY PLAN {sql}");
+    let out = Command::new(bin)
+        .arg(":memory:")
+        .arg(&full)
+        .output()
+        .unwrap();
+    String::from_utf8_lossy(&out.stdout)
+        .lines()
+        .filter(|l| !l.trim().is_empty() && !l.starts_with("QUERY PLAN"))
+        .map(|l| l.trim_start_matches(|c: char| " |`*+_-".contains(c)))
+        .collect::<Vec<_>>()
+        .join("#")
+}
+
+fn rows(bin: &str, base: &str, sql: &str) -> String {
+    let full = format!("{base} {sql}");
+    let out = Command::new(bin)
+        .arg(":memory:")
+        .arg(&full)
+        .output()
+        .unwrap();
+    String::from_utf8_lossy(&out.stdout).trim_end().to_string()
+}
+
+// `b` is indexed, `c`/`d` are not; `u`/`w` are the subquery sources.
+const BASE: &str = "CREATE TABLE t(a INTEGER PRIMARY KEY, b, c, d); CREATE INDEX tb ON t(b); \
+                    CREATE TABLE u(x,y); CREATE INDEX ux ON u(x); CREATE TABLE w(z);";
+
+#[test]
+fn in_subquery_list_subquery_matches_sqlite() {
+    if !sqlite3_available() {
+        eprintln!("sqlite3 CLI not found; skipping");
+        return;
+    }
+    let g = env!("CARGO_BIN_EXE_graphitesql");
+    for q in [
+        // NOT IN never seeks the IN column → the access stays SCAN in both.
+        "SELECT * FROM t WHERE b NOT IN (SELECT y FROM u)",
+        "SELECT * FROM t WHERE c NOT IN (SELECT z FROM w)",
+        "SELECT * FROM t WHERE b NOT IN (SELECT z FROM w WHERE z>3)",
+        "SELECT * FROM t WHERE b NOT IN (SELECT y FROM u) AND c=5",
+        // Positive IN on an UNINDEXED column → SQLite also scans.
+        "SELECT * FROM t WHERE d IN (SELECT y FROM u)",
+        "SELECT d FROM t WHERE d IN (SELECT y FROM u)",
+    ] {
+        assert_eq!(plan("sqlite3", BASE, q), plan(g, BASE, q), "plan for {q}");
+    }
+}
+
+#[test]
+fn in_subquery_out_of_subset_declines_to_bare_scan() {
+    // These render a SEARCH / different node shape in sqlite that graphite doesn't
+    // reproduce yet; graphite must keep its prior bare `SCAN t` (no LIST SUBQUERY /
+    // bloom node emitted into a non-matching plan).
+    let g = env!("CARGO_BIN_EXE_graphitesql");
+    for q in [
+        "SELECT * FROM t WHERE b IN (SELECT y FROM u)", // positive IN on indexed col → SQLite SEARCHes
+        "SELECT * FROM t WHERE a IN (SELECT x FROM u)", // rowid seek
+        "SELECT * FROM t WHERE rowid IN (SELECT x FROM u)",
+        "SELECT * FROM t WHERE b IN (SELECT y FROM u WHERE u.y=t.a)", // correlated
+        "SELECT * FROM t WHERE b NOT IN (SELECT y FROM u UNION SELECT x FROM u)", // compound body
+        "SELECT * FROM t WHERE d IN (SELECT y FROM u) AND (SELECT count(*) FROM w)>0", // cross-position
+    ] {
+        assert_eq!(plan(g, BASE, q), "SCAN t", "expected bare SCAN for {q}");
+    }
+}
+
+#[test]
+fn in_subquery_rows_unaffected() {
+    if !sqlite3_available() {
+        eprintln!("sqlite3 CLI not found; skipping");
+        return;
+    }
+    let g = env!("CARGO_BIN_EXE_graphitesql");
+    let base = format!(
+        "{BASE} INSERT INTO t VALUES(1,10,5,7),(2,20,6,8),(3,30,7,7),(4,40,8,9); \
+         INSERT INTO u VALUES(7,7),(8,8);"
+    );
+    for q in [
+        "SELECT a FROM t WHERE b NOT IN (SELECT y FROM u) ORDER BY a",
+        "SELECT a FROM t WHERE d IN (SELECT y FROM u) ORDER BY a",
+        "SELECT count(*) FROM t WHERE c NOT IN (SELECT x FROM u)",
+    ] {
+        assert_eq!(rows("sqlite3", &base, q), rows(g, &base, q), "rows for {q}");
+    }
+}
diff --git a/tests/eqp_where_scalar_subquery.rs b/tests/eqp_where_scalar_subquery.rs
@@ -173,8 +173,8 @@ fn where_scalar_subquery_declines_unrenderable() {
         "SELECT * FROM t WHERE a=(SELECT x FROM u WHERE x=a)",
         // EXISTS → CORRELATED SCALAR SUBQUERY (a different node).
         "SELECT * FROM t WHERE EXISTS(SELECT 1 FROM u WHERE x=a)",
-        // IN (SELECT) → LIST SUBQUERY + CREATE BLOOM FILTER.
-        "SELECT * FROM t WHERE b IN (SELECT x FROM u)",
+        // (`b IN (SELECT x FROM u)` now renders LIST SUBQUERY + CREATE BLOOM FILTER —
+        // see B9a and `tests/eqp_in_subquery.rs`.)
         // A CTE reference bumps the id counter → SQLite numbers it 2.
         "WITH cte AS (SELECT x FROM u) SELECT * FROM t WHERE a=(SELECT min(x) FROM cte)",
         // A compound (UNION) body bumps the counter too.
