feat(v3): N-block ORE comparator + ordered numeric & timestamptz

Derive the ORE block count N from term length instead of hardcoding 8,
and wire the ordered numeric scalar while promoting timestamptz to ordered.

Author: tobyhede

Cargo.lock

@@ -219,13 +219,15 @@ const ORDERED_INT_DOMAINS: &[DomainSpec] = &[
     },
 ];
 
-/// Equality-only domains: storage (no terms) + `_eq` (hm). Used by scalar types
-/// that can hash for equality but cannot (yet) be ordered. `timestamptz` is the
-/// first such type: cipherstash encrypts `Plaintext::Timestamp` at native
-/// 12-block ORE width, but EQL's only ORE comparator
-/// (`eql_v2.compare_ore_block_u64_8_256_term`) is hardcoded to 8 blocks, so an
-/// ordered domain would silently mis-order. Ordering is deferred until a
-/// wide-ORE (12-block) term exists.
+/// Equality-only domains: storage (no terms) + `_eq` (hm). The canonical shape
+/// for a scalar type that can hash for equality but is not ORE-orderable.
+/// **Currently unused:** `timestamptz` (the previous sole user) was promoted to
+/// the ordered shape once `eql_v3.compare_ore_block_256_term` generalized to N
+/// blocks and could order its native 12-block ORE width. Retained — and still
+/// validated as a known-valid shape by `every_type_uses_a_known_domain_shape` —
+/// so a future non-orderable scalar (e.g. a hash-only type) can reuse it without
+/// reconstructing the shape.
+#[allow(dead_code)]
 const EQ_ONLY_DOMAINS: &[DomainSpec] = &[
     DomainSpec {
         suffix: "",
@@ -285,6 +287,17 @@ const TIMESTAMPTZ_FIXTURES: &[Fixture] = fixtures!(timestamptz;
     "2012-06-30T11:59:59Z", "2016-03-15T08:15:30Z", "2020-10-21T14:45:00Z",
     "2024-02-29T17:30:45Z", "2038-01-19T03:14:07Z", "2099-12-31T23:59:59Z");
 
+/// `numeric` fixture plaintexts — distinct by `Decimal` value, spanning sign,
+/// magnitude, and scale, and including `0` plus the min/max pivots
+/// (`-1000000000000` / `1000000000000`). They mirror `ore-rs`'s own
+/// order-pinning vectors so the 14-block ORE edges (sign + high/low blocks) are
+/// exercised. Each literal is distinct by parsed value (no `"1"`/`"1.0"`
+/// aliasing) — the harness `numeric_fixtures_distinct_by_value` guard enforces
+/// this, since the zero-dep catalog only dedupes by literal string.
+const NUMERIC_FIXTURES: &[Fixture] = fixtures!(numeric;
+    "-1000000000000", "-1000000", "-1.001", "-1", "-0.5", "-0.001",
+    "0", "0.001", "0.5", "0.999999999", "1", "1.001", "1000000", "1000000000000");
+
 const INT4: ScalarSpec = ScalarSpec {
     token: "int4",
     kind: ScalarKind::I32,
@@ -321,27 +334,42 @@ pub const DATE: ScalarSpec = ScalarSpec {
     fixtures: DATE_FIXTURES,
 };
 
-/// `timestamptz` — an **equality-only** (UTC-normalized) non-integer scalar.
-/// Uses `EQ_ONLY_DOMAINS` (storage + `_eq`) rather than the four-domain ordered
-/// shape: cipherstash encrypts `Plaintext::Timestamp` at native 12-block ORE
-/// width, but EQL's only ORE comparator
-/// (`eql_v2.compare_ore_block_u64_8_256_term`) is hardcoded to 8 blocks, so an
-/// ordered timestamptz domain would silently mis-order. Ordering is deferred to
-/// a future PR that adds a wide-ORE (12-block) term. The three "pivot" fixture
-/// values are retained as equality pivots; the kind stays ordered-shaped
-/// (carries a rust type, no i128 range) so the harness can parse them.
+/// `timestamptz` — an **ordered**, UTC-normalized non-integer scalar. Uses the
+/// four-domain ordered shape (storage, `_eq`, `_ord`, `_ord_ore`): cipherstash
+/// encrypts `Plaintext::Timestamp` at native 12-block ORE width, which the
+/// generalized `eql_v3.compare_ore_block_256_term` comparator orders correctly.
+/// Values are UTC-normalized (cipherstash has no tz-preserving type) and encrypt
+/// under the `timestamp` cast.
 ///
 /// Public (like `DATE`) because the SQLx harness reads `TIMESTAMPTZ.fixtures`
-/// directly to parse the RFC3339 strings into `chrono::DateTime<Utc>` at
-/// runtime — there is no `TIMESTAMPTZ_VALUES` const (chrono is not
-/// `const`-friendly and `eql-scalars` stays zero-dep).
+/// directly to parse the RFC3339 strings into `chrono::DateTime<Utc>` at runtime
+/// (no `TIMESTAMPTZ_VALUES` const; `eql-scalars` stays zero-dep).
 pub const TIMESTAMPTZ: ScalarSpec = ScalarSpec {
     token: "timestamptz",
     kind: ScalarKind::Timestamptz,
-    domains: EQ_ONLY_DOMAINS,
+    domains: ORDERED_INT_DOMAINS,
     fixtures: TIMESTAMPTZ_FIXTURES,
 };
 
+/// `numeric` — an **ordered** non-integer scalar backed by
+/// `rust_decimal::Decimal`. Uses the four-domain ordered shape: cipherstash
+/// encrypts `Plaintext::Decimal` at native 14-block ORE width, which the
+/// generalized `eql_v3.compare_ore_block_256_term` comparator orders correctly.
+/// `numeric_value` returns `None` (no i128 range); ordering is supplied by the
+/// harness `Decimal: Ord`, which `ore-rs` guarantees agrees with the ciphertext
+/// order (equivalent scales collide, like `Decimal`'s own `Ord`).
+///
+/// Public (like `DATE` / `TIMESTAMPTZ`) so the SQLx harness reads
+/// `NUMERIC.fixtures` directly to parse the decimal strings into
+/// `rust_decimal::Decimal` at runtime (the catalog stays zero-dep: no
+/// `rust_decimal`).
+pub const NUMERIC: ScalarSpec = ScalarSpec {
+    token: "numeric",
+    kind: ScalarKind::Numeric,
+    domains: ORDERED_INT_DOMAINS,
+    fixtures: NUMERIC_FIXTURES,
+};
+
 /// Domains for `text`: the ordered shape plus a `_match` domain backed by the
 /// `Bloom` term (`@>`/`<@` containment). The ordered subset (`""`, `_eq`,
 /// `_ord_ore`, `_ord`) is identical to `ORDERED_INT_DOMAINS`; `_match` is the
@@ -396,7 +424,7 @@ pub const TEXT: ScalarSpec = ScalarSpec {
 
 /// The scalar catalog — the single source of truth. Order is significant (it
 /// drives generation order). New types are appended as their SQL surface lands.
-pub const CATALOG: &[ScalarSpec] = &[INT4, INT2, INT8, DATE, TIMESTAMPTZ, TEXT];
+pub const CATALOG: &[ScalarSpec] = &[INT4, INT2, INT8, DATE, TIMESTAMPTZ, NUMERIC, TEXT];
 
 /// Materialise an integer scalar's fixtures into a typed `&'static` slice at
 /// compile time. This is the **single-sourced** plaintext list the SQLx test

crates/eql-scalars/src/lib.rs

@@ -170,7 +170,24 @@ mod rust_tests {
         let date = CATALOG.iter().find(|s| s.token == "date").unwrap();
         assert!(!date.is_eq_only(), "date is ordered");
         let ts = CATALOG.iter().find(|s| s.token == "timestamptz").unwrap();
-        assert!(ts.is_eq_only(), "timestamptz is equality-only");
+        assert!(
+            !ts.is_eq_only(),
+            "timestamptz is now ordered (native 12-block ORE, comparator generalized to N blocks)"
+        );
+
+        // No catalog type is currently eq-only, so exercise `is_eq_only()`'s
+        // positive path with a synthetic spec built on the retained
+        // `EQ_ONLY_DOMAINS` shape (storage + `_eq`, no `_ord`).
+        let eq_only = ScalarSpec {
+            token: "synthetic_eq_only",
+            kind: ScalarKind::Timestamptz,
+            domains: EQ_ONLY_DOMAINS,
+            fixtures: &[],
+        };
+        assert!(
+            eq_only.is_eq_only(),
+            "a storage+_eq spec (no _ord) must be detected as eq-only"
+        );
     }
 }
 
@@ -496,11 +513,19 @@ mod catalog_tests {
     }
 
     #[test]
-    fn catalog_has_int4_int2_int8_date_timestamptz_text_in_order() {
+    fn catalog_has_int4_int2_int8_date_timestamptz_numeric_text_in_order() {
         let tokens: Vec<&str> = CATALOG.iter().map(|s| s.token).collect();
         assert_eq!(
             tokens,
-            vec!["int4", "int2", "int8", "date", "timestamptz", "text"]
+            vec![
+                "int4",
+                "int2",
+                "int8",
+                "date",
+                "timestamptz",
+                "numeric",
+                "text"
+            ]
         );
     }
 
@@ -607,16 +632,14 @@ mod catalog_tests {
 
     #[test]
     fn ordered_and_eq_only_shapes_are_used_as_declared() {
-        // Pin which catalog tokens carry which shape, so a row silently flipping
-        // ORDERED_INT_DOMAINS <-> EQ_ONLY_DOMAINS is caught. timestamptz is
-        // equality-only (12-block ORE vs 8-block comparator); the rest ordered
-        // (text adds a `_match` domain on top, so it is not eq_only either).
+        // All current catalog types use the four-domain ordered shape; none is
+        // equality-only. (timestamptz was promoted to ordered once the ORE
+        // comparator generalized to N blocks — see the numeric/ORE work.)
         for s in CATALOG {
             let is_eq_only = s.domains.len() == 2;
-            let expect_eq_only = s.token == "timestamptz";
-            assert_eq!(
-                is_eq_only, expect_eq_only,
-                "{} domain shape (eq_only={is_eq_only}) does not match expectation",
+            assert!(
+                !is_eq_only,
+                "{} is unexpectedly eq-only; no catalog type is eq-only currently",
                 s.token
             );
         }

crates/eql-scalars/src/tests.rs

@@ -93,6 +93,14 @@ fn is_text_token(token: &str) -> bool {
     spec_for_token(token).kind.is_text()
 }
 
+/// True when `token`'s catalog row is the `numeric` kind (owned
+/// `rust_decimal::Decimal`). Like `text` it is ordered but non-integer and
+/// non-chrono, so it stamps the `numeric` fixture discriminator and draws its
+/// values from the harness accessor (`numeric_values()`).
+fn is_numeric_token(token: &str) -> bool {
+    matches!(spec_for_token(token).kind, eql_scalars::ScalarKind::Numeric)
+}
+
 /// True when `token`'s catalog row declares no ordered domain — equality-only.
 /// Replaces the `[eq_only]` marker. Consumed by [`matrix_suite_for_entry`] to
 /// keep an eq-only type out of the ordered matrix (which exercises ordering
@@ -213,10 +221,12 @@ fn scalar_fixture_modules_tokens(list: &ScalarList) -> TokenStream2 {
                 format_ident!("temporal")
             } else if is_text_token(&token_str) {
                 format_ident!("text")
+            } else if is_numeric_token(&token_str) {
+                format_ident!("numeric")
             } else {
                 panic!(
-                    "scalar token `{token_str}` is neither integer, temporal, nor \
-                     text — no fixture discriminator is wired for its kind"
+                    "scalar token `{token_str}` is neither integer, temporal, text, \
+                     nor numeric — no fixture discriminator is wired for its kind"
                 )
             };
             quote! {

crates/eql-tests-macros/src/lib.rs

@@ -109,7 +109,16 @@ AS $$
 
     left_block_size CONSTANT smallint := 16;
     right_block_size CONSTANT smallint := 32;
-    right_offset CONSTANT smallint := 136; -- 8 * 17
+
+    -- Block count N is DERIVED from the ciphertext length, not hardcoded to 8.
+    -- Wire format per term:
+    --   [ N PRP bytes ][ N*16B left blocks ][ 16B hash key ][ N*32B right blocks ]
+    --   octet_length = 17*N + 16 + 32*N = 49*N + 16  =>  N = (octet_length - 16) / 49
+    -- This serves int4 (N=8, 408B), timestamp (N=12, 604B), and numeric
+    -- (N=14, 702B) with one comparator.
+    n            integer;
+    left_offset  integer;  -- ordinal offset of the first left block (1 + N PRP bytes)
+    right_offset integer;  -- ordinal start of the right CT (= total left CT length = 17*N)
 
     indicator smallint := 0;
   BEGIN
@@ -129,10 +138,23 @@ AS $$
       RAISE EXCEPTION 'Ciphertexts are different lengths';
     END IF;
 
-    FOR block IN 0..7 LOOP
+    -- Well-formedness: length must be exactly 49*N + 16 for some N >= 1. The
+    -- modulo alone is insufficient -- a 16-byte term passes (16 - 16) % 49 = 0
+    -- and derives N = 0, which would fall through to the all-blocks-equal path
+    -- and return 0 instead of raising. The `<= 16` clause is load-bearing.
+    IF octet_length(a.bytes) <= 16 OR (octet_length(a.bytes) - 16) % 49 != 0 THEN
+      RAISE EXCEPTION 'Malformed ORE term: % bytes', octet_length(a.bytes);
+    END IF;
+
+    n := (octet_length(a.bytes) - 16) / 49;
+    left_offset := 1 + n;     -- left blocks begin right after the N PRP bytes
+    right_offset := 17 * n;   -- right CT begins right after the 17*N-byte left CT
+
+    FOR block IN 0..n-1 LOOP
+      -- Compare each PRP byte (the first N bytes) and its 16-byte left block.
       IF
         substr(a.bytes, 1 + block, 1) != substr(b.bytes, 1 + block, 1)
-        OR substr(a.bytes, 9 + left_block_size * block, left_block_size) != substr(b.bytes, 9 + left_block_size * block, left_block_size)
+        OR substr(a.bytes, left_offset + left_block_size * block, left_block_size) != substr(b.bytes, left_offset + left_block_size * block, left_block_size)
       THEN
         IF eq THEN
           unequal_block := block;
@@ -145,11 +167,13 @@ AS $$
       RETURN 0::integer;
     END IF;
 
+    -- Hash key is the IV from the right CT of b.
     hash_key := substr(b.bytes, right_offset + 1, 16);
 
+    -- First right block is at right_offset + nonce_size (ordinally indexed).
     target_block := substr(b.bytes, right_offset + 17 + (unequal_block * right_block_size), right_block_size);
 
-    data_block := substr(a.bytes, 9 + (left_block_size * unequal_block), left_block_size);
+    data_block := substr(a.bytes, left_offset + (left_block_size * unequal_block), left_block_size);
 
     encrypt_block := encrypt(data_block::bytea, hash_key::bytea, 'aes-ecb');
 

src/v3/sem/ore_block_256/functions.sql

@@ -4,7 +4,7 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-sqlx = { version = "0.8", features = ["runtime-tokio", "postgres", "macros", "chrono"] }
+sqlx = { version = "0.8", features = ["runtime-tokio", "postgres", "macros", "chrono", "rust_decimal"] }
 tokio = { version = "1", features = ["full"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
@@ -16,6 +16,11 @@ cipherstash-client = { version = "0.35", features = ["tokio"] }
 # it as a direct dependency so the harness can name `chrono::NaiveDate` for the
 # `date` scalar (Encode/Decode/Type come from the sqlx `chrono` feature above).
 chrono = { version = "0.4", default-features = false }
+# rust_decimal backs the `numeric` scalar. The sqlx `rust_decimal` feature above
+# provides Encode/Decode/Type<Postgres>; this names the type directly for the
+# harness impls (scalar_domains.rs, eql_plaintext.rs). Already in the tree
+# transitively (cipherstash-client / ore-rs).
+rust_decimal = "1"
 paste = "1"
 eql-scalars = { path = "../../crates/eql-scalars" }
 eql-tests-macros = { path = "../../crates/eql-tests-macros" }

tests/sqlx/Cargo.toml

@@ -60,6 +60,7 @@ impl PlaintextSqlType {
     pub const TIMESTAMPTZ: PlaintextSqlType = PlaintextSqlType("timestamp with time zone");
     pub const TEXT: PlaintextSqlType = PlaintextSqlType("text");
     pub const JSONB: PlaintextSqlType = PlaintextSqlType("jsonb");
+    pub const NUMERIC: PlaintextSqlType = PlaintextSqlType("numeric");
 
     pub fn as_str(&self) -> &'static str {
         self.0
@@ -86,7 +87,8 @@ const fn cast_for_kind(kind: ScalarKind) -> Cast {
         ScalarKind::Date => Cast::DATE,
         ScalarKind::Timestamptz => Cast::TIMESTAMP,
         ScalarKind::Text => Cast::TEXT,
-        ScalarKind::Numeric | ScalarKind::Jsonb => {
+        ScalarKind::Numeric => Cast::DECIMAL,
+        ScalarKind::Jsonb => {
             panic!("EqlPlaintext is only implemented for the wired scalar kinds")
         }
     }
@@ -103,7 +105,8 @@ const fn plaintext_sql_type_for_kind(kind: ScalarKind) -> PlaintextSqlType {
         ScalarKind::Date => PlaintextSqlType::DATE,
         ScalarKind::Timestamptz => PlaintextSqlType::TIMESTAMPTZ,
         ScalarKind::Text => PlaintextSqlType::TEXT,
-        ScalarKind::Numeric | ScalarKind::Jsonb => {
+        ScalarKind::Numeric => PlaintextSqlType::NUMERIC,
+        ScalarKind::Jsonb => {
             panic!("EqlPlaintext is only implemented for the wired scalar kinds")
         }
     }
@@ -118,6 +121,7 @@ mod sealed {
     impl Sealed for chrono::DateTime<chrono::Utc> {}
     impl Sealed for String {}
     impl Sealed for serde_json::Value {}
+    impl Sealed for rust_decimal::Decimal {}
 }
 
 /// A Rust type usable as a fixture `plaintext` value, carrying its EQL cast
@@ -213,6 +217,14 @@ impl EqlPlaintext for serde_json::Value {
     }
 }
 
+impl EqlPlaintext for rust_decimal::Decimal {
+    const KIND: ScalarKind = ScalarKind::Numeric;
+
+    fn to_plaintext(&self) -> Plaintext {
+        Plaintext::Decimal(Some(*self))
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;