Merge pull request #272 from cipherstash/eql-v3-ore-cllw

feat(v3): add eql_v3.ore_cllw SEM index-term type and operators

Author: tobyhede

crates/eql-codegen/src/context.rs

@@ -268,10 +268,13 @@ pub const AGGREGATE_OPS: &[AggregateOp] = &[
     },
 ];
 
-/// True if the domain carries a comparator term (supports `<`).
-/// Port of `is_ord_capable`.
+/// True if the domain carries a comparator term (any term that supports `<`).
+/// Unions across the terms — consistent with `Term::operators_for_terms` —
+/// rather than reading only the first, so a future mixed-term domain that
+/// carries an ord term anywhere is correctly ord-capable. Port of
+/// `is_ord_capable`.
 pub fn is_ord_capable(terms: &[Term]) -> bool {
-    Term::role_for_terms(terms) == Role::Ord
+    terms.iter().any(|t| t.role() == Role::Ord)
 }
 
 #[cfg(test)]
@@ -290,6 +293,17 @@ mod tests {
         assert!(!is_ord_capable(&[]));
     }
 
+    #[test]
+    fn is_ord_capable_unions_across_terms() {
+        // An ord term anywhere in the list makes the domain ord-capable,
+        // regardless of position — consistent with operators_for_terms' union,
+        // not the first-term role. (No catalog domain is multi-term today; this
+        // pins the order-independent semantics for a future mixed-term domain.)
+        assert!(is_ord_capable(&[Term::Hm, Term::Ore]));
+        assert!(is_ord_capable(&[Term::Ore, Term::Hm]));
+        assert!(!is_ord_capable(&[Term::Hm, Term::Bloom]));
+    }
+
     #[test]
     fn environment_has_whole_file_and_partial_templates() {
         let env = environment();

crates/eql-codegen/tests/parity.rs

@@ -102,10 +102,12 @@ fn rust_generator_matches_reference_files() {
         let ref_dir = root.join("tests/codegen/reference").join(&token);
         let gen_dir = out.path().join("src/v3/scalars").join(&token);
 
-        // Assert the generated .sql file SET matches the reference set first — the
+        // Assert the generated .sql file SET matches the reference set first. The
         // per-file byte comparison below only iterates reference files, so a
-        // missing generated file (or an extra one the reference never pins) would
-        // otherwise pass silently.
+        // missing generated file would surface only as an opaque `unwrap` panic on
+        // the `read_to_string` below, and an EXTRA generated file (one the
+        // reference never pins) would pass silently — it is never iterated. This
+        // set check turns both into a clear file-set diff.
         let ref_names = sql_names(&ref_dir);
         let gen_names = sql_names(&gen_dir);
         assert_eq!(

crates/eql-scalars/src/lib.rs

@@ -86,11 +86,12 @@ pub enum Term {
     Bloom,
 }
 
-/// The generated-file role of a domain, derived from its first term (or
-/// `Storage` for a term-less domain). Gates ord-only codegen (aggregates) via an
-/// exhaustive `==` against [`Role::Ord`] rather than a stringly-typed compare —
-/// a typo can no longer silently disable aggregate generation. `label` is the
-/// `&'static str` form for any future template/serde consumer.
+/// The generated-file role of a domain, resolved from its terms by the
+/// richest-comparison precedence in [`Role::rank`] (or `Storage` for a term-less
+/// domain). Gates ord-only codegen (aggregates) via an exhaustive `==` against
+/// [`Role::Ord`] rather than a stringly-typed compare — a typo can no longer
+/// silently disable aggregate generation. `label` is the `&'static str` form for
+/// any future template/serde consumer.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Role {
     Storage,
@@ -109,6 +110,23 @@ impl Role {
             Role::Match => "match",
         }
     }
+
+    /// Precedence used by [`Term::role_for_terms`] to resolve a multi-term
+    /// domain to a single generated-file role: the richest comparison capability
+    /// wins (`Ord > Eq > Match > Storage`). Ordering subsumes equality, so an
+    /// `Ore` term anywhere makes the domain ord-shaped; `Match` (containment) is
+    /// a weaker standalone surface; `Storage` is the absence of any term. The
+    /// current catalog is single-term, so this only disambiguates a hypothetical
+    /// future mixed-term domain — and keeps `role_for_terms` consistent with
+    /// [`Term::operators_for_terms`], which already unions across all terms.
+    pub const fn rank(self) -> u8 {
+        match self {
+            Role::Storage => 0,
+            Role::Match => 1,
+            Role::Eq => 2,
+            Role::Ord => 3,
+        }
+    }
 }
 
 /// A single fixture plaintext value, value-kind tagged: `Min`/`Max`/`Zero` are

crates/eql-scalars/src/term.rs

@@ -33,7 +33,8 @@ impl Term {
         }
     }
 
-    /// Generated-file [`Role`] for a domain whose first term is this one.
+    /// Generated-file [`Role`] contributed by this single term. A domain's role
+    /// is the richest of its terms' roles — see [`Term::role_for_terms`].
     pub const fn role(self) -> Role {
         match self {
             Term::Hm => Role::Eq,
@@ -117,11 +118,18 @@ impl Term {
     }
 
     /// Generated-file [`Role`] for a domain with these terms. No terms =>
-    /// [`Role::Storage`]; otherwise the first term's role.
+    /// [`Role::Storage`]; otherwise the **richest** role across the terms by
+    /// [`Role::rank`] precedence (`Ord > Eq > Match > Storage`). For the current
+    /// single-term catalog this equals the lone term's role, so generated SQL is
+    /// unchanged; the precedence only disambiguates a future mixed-term domain
+    /// (e.g. `[Hm, Ore]` => `Ord`), keeping this consistent with
+    /// [`Term::operators_for_terms`], which unions across all terms rather than
+    /// reading only the first.
     pub fn role_for_terms(terms: &[Term]) -> Role {
-        match terms.first() {
-            None => Role::Storage,
-            Some(t) => t.role(),
-        }
+        terms
+            .iter()
+            .map(|t| t.role())
+            .max_by_key(|r| r.rank())
+            .unwrap_or(Role::Storage)
     }
 }

crates/eql-scalars/src/tests.rs

@@ -292,6 +292,26 @@ mod term_helper_tests {
         assert_eq!(Term::role_for_terms(&[Term::Ore]), Role::Ord);
     }
 
+    #[test]
+    fn role_for_terms_takes_richest_role_order_independently() {
+        // A mixed-term domain resolves to the richest role by Role::rank
+        // precedence (Ord > Eq > Match > Storage), regardless of term order —
+        // consistent with operators_for_terms' union, not the first term. No
+        // catalog domain is multi-term today; this pins the semantics so a future
+        // `[Hm, Ore]` domain generates the ord surface (and its aggregates).
+        assert_eq!(Term::role_for_terms(&[Term::Hm, Term::Ore]), Role::Ord);
+        assert_eq!(Term::role_for_terms(&[Term::Ore, Term::Hm]), Role::Ord);
+        assert_eq!(Term::role_for_terms(&[Term::Hm, Term::Bloom]), Role::Eq);
+        assert_eq!(Term::role_for_terms(&[Term::Bloom, Term::Hm]), Role::Eq);
+    }
+
+    #[test]
+    fn role_rank_orders_richest_comparison_highest() {
+        assert!(Role::Ord.rank() > Role::Eq.rank());
+        assert!(Role::Eq.rank() > Role::Match.rank());
+        assert!(Role::Match.rank() > Role::Storage.rank());
+    }
+
     #[test]
     fn extractor_terms_dedupes_by_extractor_first_occurrence_wins() {
         // No catalog domain currently carries two terms sharing an extractor, so

mise.toml

@@ -164,23 +164,28 @@ description = "Verify the matrix test-name set against the single canonical snap
 dir = "{{config_root}}/tests/sqlx"
 run = """
 #!/usr/bin/env bash
-# ONE canonical, token-normalized snapshot (snapshots/matrix_tests.txt) pins the
-# set of macro-emitted matrix test names for the ORDERED scalar shape. There is
-# no second committed file for the equality-only shape: an eq-only type's name
-# set is exactly the ordered set MINUS the ord-only lines, so the inventory
-# DERIVES it from the one baseline (ordered minus `_ord`/`order_by`/
-# `routes_through_ob`). Each discovered type must match either the full baseline
-# (ordered) or that derived subset (eq-only). This keeps one committed snapshot
-# however many ordered/eq-only types exist.
+# Two committed, token-normalized snapshots. The canonical one
+# (snapshots/matrix_tests.txt) pins the set of macro-emitted matrix test names
+# for the ORDERED scalar shape. The second (snapshots/matrix_tests_eq_only.txt)
+# is the equality-only shape: an eq-only type's name set is exactly the ordered
+# set MINUS the ord-only lines (`_ord`/`order_by`/`routes_through_ob`), so it is
+# DERIVED from the one baseline — but it is also committed and PINNED: the gate
+# re-derives the subset at runtime and asserts it equals the committed file
+# (step 3 below), so a change to the baseline or the strip filter that alters the
+# eq-only set must be re-committed deliberately. Each discovered type must then
+# match either the full baseline (ordered) or that derived/pinned subset
+# (eq-only). One baseline drives both shapes however many types exist.
 #
 # Steps:
 #   1. List the encrypted_domain binary ONCE (deterministic; reused below).
 #   2. Discover the set of scalar types present FROM THE BINARY'S OWN OUTPUT
 #      (scalars::<X>:: prefixes) — never a directory glob.
-#   3. For each discovered type, normalize its token to <T> and assert its set
+#   3. Derive the eq-only subset from the ordered baseline and assert it equals
+#      the committed snapshots/matrix_tests_eq_only.txt (pins the derivation).
+#   4. For each discovered type, normalize its token to <T> and assert its set
 #      equals EITHER the canonical snapshot (ordered) OR the derived eq-only
 #      subset. Assert at least one type is present.
-#   4. Completeness cross-check: assert the discovered type set equals
+#   5. Completeness cross-check: assert the discovered type set equals
 #      `eql-codegen list-types`. A catalog type added without its matrix wiring
 #      (no scalars::<T>:: tests in the binary) fails here.
 #

src/v3/sem/ore_cllw/functions.sql

@@ -0,0 +1,171 @@
+-- REQUIRE: src/v3/schema.sql
+-- REQUIRE: src/v3/sem/ore_cllw/types.sql
+
+--! @file v3/sem/ore_cllw/functions.sql
+--! @brief CLLW ORE index-term extraction and comparison (eql_v3 SEM).
+
+--! @brief Extract CLLW ORE index term from raw jsonb
+--!
+--! Returns the CLLW ORE ciphertext from the `oc` field of a single sv element
+--! supplied as raw jsonb. Inlinable single-statement SQL — the planner folds
+--! the body into the calling query.
+--!
+--! **Missing-`oc` semantics**: returns SQL-level NULL (not a composite with
+--! NULL bytes) when `oc` is absent, so btree's NULL handling filters those
+--! rows from range queries.
+--!
+--! @param val jsonb An object carrying an `oc` field
+--! @return eql_v3.ore_cllw Composite carrying the CLLW ciphertext, or NULL
+--!         when the `oc` field is absent.
+--! @see eql_v3.has_ore_cllw
+--! @see eql_v3.compare_ore_cllw_term
+CREATE FUNCTION eql_v3.ore_cllw(val jsonb)
+  RETURNS eql_v3.ore_cllw
+  LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+  SELECT CASE WHEN val ->> 'oc' IS NULL THEN NULL
+              ELSE ROW(decode(val ->> 'oc', 'hex'))::eql_v3.ore_cllw
+         END
+$$;
+
+COMMENT ON FUNCTION eql_v3.ore_cllw(jsonb) IS
+  'eql-inline-critical: raw-jsonb CLLW extractor; must stay inlinable (unpinned search_path)';
+
+--! @brief Check if a raw jsonb value contains a CLLW ORE index term
+--! @param val jsonb An object that may carry an `oc` field
+--! @return boolean True if `oc` field is present and non-null
+CREATE FUNCTION eql_v3.has_ore_cllw(val jsonb)
+  RETURNS boolean
+  LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+  SELECT val ->> 'oc' IS NOT NULL
+$$;
+
+COMMENT ON FUNCTION eql_v3.has_ore_cllw(jsonb) IS
+  'eql-inline-critical: raw-jsonb CLLW presence helper; must stay inlinable (unpinned search_path)';
+
+--! @brief CLLW per-byte comparison helper
+--! @internal
+--!
+--! Byte-by-byte comparison implementing the CLLW order-revealing protocol.
+--! Identify the index of the first differing byte; if `(y_byte + 1) == x_byte`
+--! (mod 256) there, then x > y; otherwise x < y. Equal inputs return 0. Inputs
+--! MUST be the same length (the caller guarantees this). Stays `LANGUAGE
+--! plpgsql` — the per-byte loop can't be a single inlinable SQL expression.
+--!
+--! @param a bytea First CLLW ciphertext slice
+--! @param b bytea Second CLLW ciphertext slice
+--! @return integer -1, 0, or 1
+--! @throws Exception if inputs are different lengths
+--! @see eql_v3.compare_ore_cllw_term
+CREATE FUNCTION eql_v3.compare_ore_cllw_term_bytes(a bytea, b bytea)
+RETURNS int
+  SET search_path = pg_catalog, extensions, public
+AS $$
+DECLARE
+    len_a INT;
+    len_b INT;
+    i INT;
+    first_diff INT := 0;
+BEGIN
+
+    len_a := LENGTH(a);
+    len_b := LENGTH(b);
+
+    IF len_a != len_b THEN
+      RAISE EXCEPTION 'ore_cllw index terms are not the same length';
+    END IF;
+
+    FOR i IN 1..len_a LOOP
+        IF first_diff = 0 AND get_byte(a, i - 1) != get_byte(b, i - 1) THEN
+            first_diff := i;
+        END IF;
+    END LOOP;
+
+    IF first_diff = 0 THEN
+        RETURN 0;
+    END IF;
+
+    IF ((get_byte(b, first_diff - 1) + 1) & 255) = get_byte(a, first_diff - 1) THEN
+        RETURN 1;
+    ELSE
+        RETURN -1;
+    END IF;
+END;
+$$ LANGUAGE plpgsql;
+
+--! @brief Variable-length CLLW ORE term comparison
+--! @internal
+--!
+--! Three-way comparison of two CLLW ORE ciphertext terms of potentially
+--! different lengths. Compares the shared prefix via the CLLW per-byte
+--! protocol; on equal prefixes, the shorter input sorts first. The leading
+--! domain-tag byte makes numeric (`0x00`) sort before string (`0x01`). Stays
+--! `LANGUAGE plpgsql` because it dispatches to `compare_ore_cllw_term_bytes`.
+--!
+--! btree filters NULL composites at the row level, so this should never see a
+--! NULL composite under normal operation; the IS-NULL guard returns NULL
+--! defensively. A non-NULL composite with NULL `bytes` is a contract violation
+--! — the extractor returns SQL NULL (not ROW(NULL)) on missing `oc`, so raise
+--! loudly rather than silently misorder.
+--!
+--! @param a eql_v3.ore_cllw First term
+--! @param b eql_v3.ore_cllw Second term
+--! @return integer -1, 0, or 1; NULL if either composite is NULL
+--! @throws Exception if either composite has a NULL `bytes` field
+--! @see eql_v3.compare_ore_cllw_term_bytes
+CREATE FUNCTION eql_v3.compare_ore_cllw_term(a eql_v3.ore_cllw, b eql_v3.ore_cllw)
+RETURNS int
+  SET search_path = pg_catalog, extensions, public
+AS $$
+DECLARE
+    len_a INT;
+    len_b INT;
+    common_len INT;
+    cmp_result INT;
+BEGIN
+    IF a::text IS NULL OR b::text IS NULL THEN
+      RETURN NULL;
+    END IF;
+
+    IF a.bytes IS NULL OR b.bytes IS NULL THEN
+      RAISE EXCEPTION 'eql_v3.compare_ore_cllw_term: composite has NULL bytes field — extractor invariant violated. Check that the index expression uses eql_v3.ore_cllw(...) and not a hand-crafted ROW(NULL).';
+    END IF;
+
+    len_a := LENGTH(a.bytes);
+    len_b := LENGTH(b.bytes);
+
+    IF len_a = 0 AND len_b = 0 THEN
+        RETURN 0;
+    ELSIF len_a = 0 THEN
+        RETURN -1;
+    ELSIF len_b = 0 THEN
+        RETURN 1;
+    END IF;
+
+    IF len_a < len_b THEN
+        common_len := len_a;
+    ELSE
+        common_len := len_b;
+    END IF;
+
+    cmp_result := eql_v3.compare_ore_cllw_term_bytes(
+      SUBSTRING(a.bytes FROM 1 FOR common_len),
+      SUBSTRING(b.bytes FROM 1 FOR common_len)
+    );
+
+    IF cmp_result = -1 THEN
+        RETURN -1;
+    ELSIF cmp_result = 1 THEN
+        RETURN 1;
+    END IF;
+
+    IF len_a < len_b THEN
+        RETURN -1;
+    ELSIF len_a > len_b THEN
+        RETURN 1;
+    ELSE
+        RETURN 0;
+    END IF;
+END;
+$$ LANGUAGE plpgsql;

src/v3/sem/ore_cllw/operator_class.sql

@@ -0,0 +1,29 @@
+-- REQUIRE: src/v3/schema.sql
+-- REQUIRE: src/v3/sem/ore_cllw/types.sql
+-- REQUIRE: src/v3/sem/ore_cllw/functions.sql
+-- REQUIRE: src/v3/sem/ore_cllw/operators.sql
+
+--! @file v3/sem/ore_cllw/operator_class.sql
+--! @brief Btree operator class on the eql_v3.ore_cllw composite type.
+--!
+--! DEFAULT FOR TYPE so a functional btree index on eql_v3.ore_cllw(expr)
+--! engages without an explicit opclass annotation. FUNCTION 1 is the three-way
+--! comparator btree's internal sort uses; it is plpgsql by design (per-byte
+--! CLLW protocol needs iteration) and is called once per index-entry pair
+--! during build / search, not per-row in the outer query.
+--!
+--! @note Excluded from the Supabase build variant by the build glob
+--!       `**/*operator_class.sql`.
+--! @see eql_v3.compare_ore_cllw_term
+
+CREATE OPERATOR FAMILY eql_v3.ore_cllw_ops USING btree;
+
+CREATE OPERATOR CLASS eql_v3.ore_cllw_ops
+  DEFAULT FOR TYPE eql_v3.ore_cllw
+  USING btree FAMILY eql_v3.ore_cllw_ops AS
+    OPERATOR 1 <  (eql_v3.ore_cllw, eql_v3.ore_cllw),
+    OPERATOR 2 <= (eql_v3.ore_cllw, eql_v3.ore_cllw),
+    OPERATOR 3 =  (eql_v3.ore_cllw, eql_v3.ore_cllw),
+    OPERATOR 4 >= (eql_v3.ore_cllw, eql_v3.ore_cllw),
+    OPERATOR 5 >  (eql_v3.ore_cllw, eql_v3.ore_cllw),
+    FUNCTION 1 eql_v3.compare_ore_cllw_term(eql_v3.ore_cllw, eql_v3.ore_cllw);