Merge pull request #301 from cipherstash/eql_v3_scale_index_engagement

test(v3): scaled, cost-chosen index-engagement tests for encrypted-domain surface

Author: tobyhede

.github/workflows/bench-eql.yml

@@ -61,6 +61,17 @@ jobs:
           mise run postgres:up postgres-${POSTGRES_VERSION} --extra-args "--detach --wait"
 
       - name: Run bench tests
+        # CS_* scoped to THIS step only (least privilege): test:bench -> test:sqlx:prep
+        # -> fixture:generate:all encrypts via cipherstash-client and needs BOTH a
+        # ZeroKMS auth credential (CS_CLIENT_ACCESS_KEY + CS_WORKSPACE_CRN) AND a client
+        # key (CS_CLIENT_ID + CS_CLIENT_KEY); without them it fails "Auth strategy error:
+        # Not authenticated". Kept off job scope so checkout/mise/rust-cache actions
+        # never see them.
+        env:
+          CS_CLIENT_ACCESS_KEY: ${{ secrets.CS_CLIENT_ACCESS_KEY }}
+          CS_WORKSPACE_CRN: ${{ secrets.CS_WORKSPACE_CRN }}
+          CS_CLIENT_ID: ${{ secrets.CS_CLIENT_ID }}
+          CS_CLIENT_KEY: ${{ secrets.CS_CLIENT_KEY }}
         run: |
           export active_rust_toolchain=$(rustup show active-toolchain | cut -d' ' -f1)
           rustup component add --toolchain ${active_rust_toolchain} rustfmt clippy

tasks/test/bench.sh

@@ -15,14 +15,15 @@ echo "=========================================="
 
 "$(dirname "$0")/../postgres/check_container.sh" "${POSTGRES_VERSION}"
 
-echo "Building EQL..."
-mise run --output prefix --force build
-
-echo "Updating SQLx migrations with built EQL..."
-cp release/cipherstash-encrypt.sql tests/sqlx/migrations/001_install_eql.sql
-
-echo "Running SQLx migrations..."
-(cd tests/sqlx && sqlx migrate run)
+# Prep the SQLx test DB exactly like the standard suite (test:sqlx): build EQL,
+# copy it into migrations, migrate, AND regenerate the gitignored per-type
+# fixtures. The fixtures are include_str!'d into the test binary at COMPILE time
+# by #[sqlx::test(fixtures(...))], so they MUST exist on disk before `cargo test`
+# compiles. This script previously hand-rolled build+cp+migrate but omitted
+# fixture generation; once fixtures became generated/gitignored the bench binary
+# stopped compiling (couldn't read tests/sqlx/fixtures/eql_v2_*.sql). Reusing
+# prep keeps bench in lockstep with test:sqlx and prevents that drift recurring.
+mise run --output prefix test:sqlx:prep
 
 echo "Running bench tests (cargo test --features bench)..."
 (cd tests/sqlx && cargo test --features bench)

tests/sqlx/Cargo.toml

@@ -36,16 +36,19 @@ workspace = true
 [features]
 default = []
 # Opt-in to slow benchmark / regression / scale tests. Without this feature
-# they're #[ignore]'d so PR CI stays fast. The `bench-eql` workflow enables
-# it on push to main and on a nightly schedule. Run locally with:
-#   mise run test:bench
-bench = []
-# Opt-in to the matrix's per-(variant, index) scale tests. Each builds
-# ~5000 rows of filler plus a single selective pivot and asserts the
-# planner *prefers* the functional index with `enable_seqscan` left on.
-# The default index tests force seqscan off and only prove the index is
-# *usable*. Off by default to keep `mise run test` fast; CI runs with
-# `--features scale`.
+# they're #[ignore]'d or #[cfg]'d out so PR CI stays fast. Enabling `bench`
+# transitively enables `scale` (below), so the `bench-eql` workflow — push to
+# main + nightly, via `tasks/test/bench.sh` (`cargo test --features bench`) —
+# is the runner that exercises the per-combo scale-preference matrix tests.
+# Run locally with: mise run test:bench
+bench = ["scale"]
+# The matrix's per-(variant, index) scale-preference tests
+# (`#[cfg(feature = "scale")]`). Each replicates ONE real fixture payload to
+# ~5000 rows plus a selective pivot and asserts the planner *prefers* the
+# functional index with `enable_seqscan` left ON. The `*_index_engages_*` arms
+# force seqscan off and only prove the index is *usable*. Off in fast PR CI
+# (`mise run test`); activated transitively by `bench` (above), so the
+# `bench-eql` workflow runs them. Not enabled directly anywhere else.
 scale = []
 # Opt-in to the e2e property suite (CIP-3141). It generates fresh random
 # plaintexts each run and encrypts them end-to-end through ZeroKMS via

tests/sqlx/src/matrix.rs

@@ -1587,42 +1587,137 @@ macro_rules! __scalar_matrix_scale_case {
                 );
 
                 let values: &[$scalar] = <$scalar as ScalarType>::fixture_values();
-                anyhow::ensure!(values.len() >= 2,
-                    "scale test requires >= 2 fixture rows for distinct filler/pivot");
-                let filler = values[0].clone();
-                let pivot = values[values.len() / 2].clone();
-                let filler_payload =
-                    $crate::scalar_domains::fetch_fixture_payload::<$scalar>(&pool, filler).await?;
-                let pivot_payload =
-                    $crate::scalar_domains::fetch_fixture_payload::<$scalar>(&pool, pivot).await?;
+                // Distinct, sorted fixture values so MIN / MID / MAX are well
+                // defined regardless of fixture order. ONE data shape serves
+                // every op-class a combo can carry — equality combos hold `=`;
+                // the ordered combos hold `=` plus `<`/`<=`/`>`/`>=`, all sharing
+                // a single extractor (so one functional index serves them):
+                //
+                //   5000 identical MID rows (the bulk) + ONE MIN row + ONE MAX
+                //   row = 5002 rows.
+                //
+                // Each op then anchors its predicate so EXACTLY ONE row matches,
+                // making the predicate ~1/5002 selective and the functional index
+                // the cheap plan with `enable_seqscan` left ON (Fact 4). A single
+                // MIN-bulk table cannot do this for both range directions at once
+                // (`value > MIN` would match every non-MIN row); a MID bulk with
+                // one MIN and one MAX pivot makes every op single-row-selective:
+                //   `=`  anchor MIN -> the single MIN row (bulk is MID)
+                //   `<`  anchor MID -> the single MIN row (MID < MID is false)
+                //   `<=` anchor MIN -> the single MIN row
+                //   `>`  anchor MID -> the single MAX row
+                //   `>=` anchor MAX -> the single MAX row
+                let mut sorted: Vec<$scalar> = values.to_vec();
+                sorted.sort();
+                sorted.dedup();
+                anyhow::ensure!(sorted.len() >= 3,
+                    "scale test requires >= 3 distinct fixture values for \
+min/mid/max single-row selectivity");
+                let min_v = sorted[0].clone();
+                let max_v = sorted[sorted.len() - 1].clone();
+                let mid_v = sorted[sorted.len() / 2].clone();
+
+                let min_payload =
+                    $crate::scalar_domains::fetch_fixture_payload::<$scalar>(&pool, min_v).await?;
+                let mid_payload =
+                    $crate::scalar_domains::fetch_fixture_payload::<$scalar>(&pool, mid_v).await?;
+                let max_payload =
+                    $crate::scalar_domains::fetch_fixture_payload::<$scalar>(&pool, max_v).await?;
 
                 let mut tx = pool.begin().await?;
                 sqlx::query(&format!(
                     "CREATE TEMP TABLE {table} (value {d}) ON COMMIT DROP",
                 )).execute(&mut *tx).await?;
+                // The bulk: 5000 identical MID rows.
                 sqlx::query(&format!(
                     "INSERT INTO {table}(value) \
 SELECT $1::jsonb::{d} FROM generate_series(1, 5000)",
-                )).bind(&filler_payload).execute(&mut *tx).await?;
+                )).bind(&mid_payload).execute(&mut *tx).await?;
+                // The two selective pivots: exactly one MIN row and one MAX row.
                 sqlx::query(&format!(
-                    "INSERT INTO {table}(value) VALUES ($1::jsonb::{d})",
-                )).bind(&pivot_payload).execute(&mut *tx).await?;
+                    "INSERT INTO {table}(value) VALUES ($1::jsonb::{d}), ($2::jsonb::{d})",
+                )).bind(&min_payload).bind(&max_payload).execute(&mut *tx).await?;
                 sqlx::query(&format!(
                     "CREATE INDEX {index} ON {table} USING {using} ({extractor}(value))", using = $using, extractor = extractor,
                 )).execute(&mut *tx).await?;
                 sqlx::query(&format!("ANALYZE {table}"))
                     .execute(&mut *tx).await?;
-
-                let lit = pivot_payload.replace('\'', "''");
-                $crate::matrix::assert_index_scan_uses(
-                    &mut *tx,
-                    &format!("SELECT * FROM {table} WHERE value = '{lit}'::jsonb::{d}"),
-                    index,
-                    &format!(
-                        "with seqscan enabled the planner must prefer the {extractor} {using} index for a selective =",
-                        extractor = extractor, using = $using,
-                    ),
-                ).await?;
+                // enable_seqscan LEFT ON — this is the cost-PREFERENCE proof, not
+                // the usability proof (the sibling `*_index_engages_*` arm forces
+                // seqscan off over the ~17-row fixture). See Fact 1 / Fact 4.
+
+                // Both RHS forms (`::{domain}` and bare `::jsonb`) and BOTH the
+                // natural operator form and the explicit extractor form are
+                // asserted per op, mirroring the validity arm
+                // (`__scalar_matrix_index_case!`) minus the forced seqscan-off.
+                let rhs_casts = [format!("::{d}", d = d), String::new()];
+                $(
+                    // `<>` is never index-selective over 5000 rows and is not a
+                    // member of any index combo; guard it out defensively.
+                    if $op != "<>" {
+                        // Per-op anchor giving a single-row match against the
+                        // bulk-MID / one-MIN / one-MAX table (see the header).
+                        let anchor: &str = match $op {
+                            "=" => &min_payload,
+                            "<" => &mid_payload,
+                            "<=" => &min_payload,
+                            ">" => &mid_payload,
+                            ">=" => &max_payload,
+                            _ => &min_payload,
+                        };
+                        let lit = anchor.replace('\'', "''");
+                        for rhs_cast in &rhs_casts {
+                            // Natural bare-operator form: `value {op} <lit>`. This
+                            // is the inlinability tripwire — a broken inline flips
+                            // it to Seq Scan.
+                            let natural = format!(
+                                "SELECT * FROM {table} WHERE value {op} '{lit}'::jsonb{cast}",
+                                op = $op, cast = rhs_cast,
+                            );
+                            $crate::matrix::assert_index_scan_uses(
+                                &mut *tx, &natural, index,
+                                &format!(
+                                    "scale: natural-form `{op}` (rhs {cast:?}) must PREFER the \
+{extractor} {using} index for a single-row predicate (seqscan ON)",
+                                    op = $op, cast = rhs_cast,
+                                    extractor = extractor, using = $using,
+                                ),
+                            ).await?;
+
+                            // Explicit extractor form: `{extractor}(value) {op}
+                            // {extractor}(<lit>)`. Complements the natural form;
+                            // a divergence between the two surfaces an inlining
+                            // break.
+                            //
+                            // ONLY the domain-cast RHS (`::{d}`) — never bare
+                            // `::jsonb`. A standalone `eq_term`/`ord_term` call on
+                            // a bare-jsonb argument is ambiguous: the extractor is
+                            // overloaded across the domain family, and bare jsonb
+                            // implicitly casts to several of them, so Postgres
+                            // raises `function eql_v3.<extractor>(jsonb) is not
+                            // unique`. The natural operator form above already
+                            // exercises the bare-jsonb RHS path (the operator
+                            // signature pins the domain), so skipping it here loses
+                            // no coverage.
+                            if !rhs_cast.is_empty() {
+                                let extracted = format!(
+                                    "SELECT * FROM {table} \
+WHERE {extractor}(value) {op} {extractor}('{lit}'::jsonb{cast})",
+                                    extractor = extractor, op = $op, cast = rhs_cast,
+                                );
+                                $crate::matrix::assert_index_scan_uses(
+                                    &mut *tx, &extracted, index,
+                                    &format!(
+                                        "scale: extractor-form `{op}` (rhs {cast:?}) must PREFER the \
+{extractor} {using} index for a single-row predicate (seqscan ON)",
+                                        op = $op, cast = rhs_cast,
+                                        extractor = extractor, using = $using,
+                                    ),
+                                ).await?;
+                            }
+                        }
+                    }
+                )+
 
                 tx.commit().await?;
                 Ok(())

tests/sqlx/tests/v3_jsonb_tests.rs

@@ -1148,6 +1148,122 @@ async fn v3_jsonb_index_to_ste_vec_query_gin_engages(pool: PgPool) -> anyhow::Re
     Ok(())
 }
 
+// ============================================================================
+// D11-scale — jsonb containment GIN is COST-CHOSEN at scale (seqscan ON).
+//
+// The sibling `v3_jsonb_index_to_ste_vec_query_gin_engages` (above) forces
+// `enable_seqscan = off` over the 10-row fixture: it proves the GIN index is
+// USABLE, not that the planner PREFERS it. This test replicates ONE real
+// fixture document to 5000 rows (the bulk) plus a single DISTINCT pivot
+// document and, leaving `enable_seqscan` ON, asserts the planner CHOOSES the
+// GIN index for a single-row-selective containment needle. Same pattern as the
+// scalar `*_scale_preference_*` arms, and `#[cfg(feature = "scale")]` so it
+// rides the bench workflow, not fast PR CI (matches the scalar scale arms).
+//
+// Real ciphertext only: both documents come from the generated `v3_ste_vec`
+// fixture, replicated via generate_series — no new fixture, no static blob.
+// Selectivity comes from the distinct-per-row `$.hello` oc leaf
+// (`SEL_HELLO_OC`, whose load-bearing distinctness is asserted by
+// `v3_jsonb_containment_oc_only` / `v3_jsonb_fixture_structural_invariants`):
+// the pivot's own oc term matches ONLY the pivot row, never the 5000 bulk rows
+// (whose oc term is the filler document's, a different value). A precondition
+// check below fails loudly if the two leaves ever collide.
+// ============================================================================
+
+#[cfg(feature = "scale")]
+#[sqlx::test(fixtures(path = "../fixtures", scripts("v3_ste_vec")))]
+async fn v3_jsonb_to_ste_vec_query_gin_is_cost_chosen(pool: PgPool) -> anyhow::Result<()> {
+    // Two DISTINCT real fixture rows: the filler (bulk) and the pivot. Their
+    // `$.hello` oc leaves differ (distinct per row), so a needle for the
+    // pivot's oc isolates exactly the single pivot row.
+    let filler_payload: String = sqlx::query_scalar(
+        "SELECT payload::jsonb::text FROM fixtures.v3_ste_vec ORDER BY id ASC LIMIT 1",
+    )
+    .fetch_one(&pool)
+    .await?;
+    let pivot_payload: String = sqlx::query_scalar(
+        "SELECT payload::jsonb::text FROM fixtures.v3_ste_vec ORDER BY id DESC LIMIT 1",
+    )
+    .fetch_one(&pool)
+    .await?;
+
+    // The pivot's own `$.hello` oc term — the same extraction the oc-containment
+    // oracle (`v3_jsonb_containment_oc_only`) uses — which the needle searches
+    // for. The filler's oc term is extracted only to assert the two differ.
+    let pivot_oc: String = sqlx::query_scalar(&format!(
+        "SELECT (payload ->> '{SEL_HELLO_OC}'::text)::jsonb ->> 'oc' \
+         FROM fixtures.v3_ste_vec ORDER BY id DESC LIMIT 1"
+    ))
+    .fetch_one(&pool)
+    .await?;
+    let filler_oc: String = sqlx::query_scalar(&format!(
+        "SELECT (payload ->> '{SEL_HELLO_OC}'::text)::jsonb ->> 'oc' \
+         FROM fixtures.v3_ste_vec ORDER BY id ASC LIMIT 1"
+    ))
+    .fetch_one(&pool)
+    .await?;
+    anyhow::ensure!(
+        filler_oc != pivot_oc,
+        "fixture precondition: filler and pivot rows must have distinct $.hello oc \
+         leaves for single-row selectivity (distinct-per-row oc is the load-bearing \
+         W1 invariant); got identical terms"
+    );
+
+    let mut tx = pool.begin().await?;
+    sqlx::query("CREATE TEMP TABLE v3_jsonb_scale (payload eql_v3.json) ON COMMIT DROP")
+        .execute(&mut *tx)
+        .await?;
+    // The bulk: 5000 copies of the filler document.
+    sqlx::query(
+        "INSERT INTO v3_jsonb_scale(payload) \
+         SELECT $1::jsonb::eql_v3.json FROM generate_series(1, 5000)",
+    )
+    .bind(&filler_payload)
+    .execute(&mut *tx)
+    .await?;
+    // The single selective pivot document.
+    sqlx::query("INSERT INTO v3_jsonb_scale(payload) VALUES ($1::jsonb::eql_v3.json)")
+        .bind(&pivot_payload)
+        .execute(&mut *tx)
+        .await?;
+    sqlx::query(
+        "CREATE INDEX v3_jsonb_scale_gin_idx ON v3_jsonb_scale \
+         USING gin ((eql_v3.to_ste_vec_query(payload)::jsonb) jsonb_path_ops)",
+    )
+    .execute(&mut *tx)
+    .await?;
+    sqlx::query("ANALYZE v3_jsonb_scale")
+        .execute(&mut *tx)
+        .await?;
+    // enable_seqscan LEFT ON — this is the cost-PREFERENCE proof, not the
+    // usability proof (the sibling `*_gin_engages` arm forces seqscan off).
+
+    // Selective needle: the pivot's own `$.hello` oc leaf. With distinct-per-row
+    // oc, exactly the single pivot row contains it.
+    let n = needle(&[(SEL_HELLO_OC, "oc", &pivot_oc)]);
+    let query =
+        format!("SELECT count(*) FROM v3_jsonb_scale WHERE payload @> '{n}'::eql_v3.ste_vec_query");
+    assert_index_scan_uses(
+        &mut *tx,
+        &query,
+        "v3_jsonb_scale_gin_idx",
+        "jsonb containment `@>` must PREFER the to_ste_vec_query GIN index at scale (seqscan ON)",
+    )
+    .await?;
+
+    // Row floor + selectivity: exactly the single pivot row matches (not zero —
+    // which would make the index-scan-over-nothing pass vacuously — and not the
+    // bulk, which would mean the needle was not selective).
+    let matched: i64 = sqlx::query_scalar(&query).fetch_one(&mut *tx).await?;
+    assert_eq!(
+        matched, 1,
+        "the GIN-engaged containment needle must match exactly the single pivot row"
+    );
+
+    tx.rollback().await?;
+    Ok(())
+}
+
 #[sqlx::test(fixtures(path = "../fixtures", scripts("v3_ste_vec")))]
 async fn v3_jsonb_index_ore_cllw_btree_engages(pool: PgPool) -> anyhow::Result<()> {
     let mut tx = pool.begin().await?;