Merge pull request #355 from cipherstash/jsonb-term-filters

feat(ste-vec): add term_filters support with integration tests

Author: tobyhede

packages/cipherstash-proxy-integration/src/common.rs

@@ -371,6 +371,82 @@ pub async fn assert_encrypted_jsonb(id: i64, plaintext: &Value) {
     }
 }
 
+/// Insert a JSON value into the encrypted_jsonb_filtered column (with downcase term filter).
+pub async fn insert_jsonb_filtered() -> (i64, Value) {
+    let id = random_id();
+
+    let encrypted_jsonb = serde_json::json!({
+        "id": id,
+        "name": "John",
+        "city": "Melbourne",
+        "nested": {
+            "title": "Engineer",
+            "department": "Technology",
+        },
+        "tags": ["Hello", "World"],
+    });
+
+    let sql = "INSERT INTO encrypted (id, encrypted_jsonb_filtered) VALUES ($1, $2)".to_string();
+
+    insert(&sql, &[&id, &encrypted_jsonb]).await;
+
+    // Verify encryption actually occurred
+    assert_encrypted_jsonb_filtered(id, &encrypted_jsonb).await;
+
+    (id, encrypted_jsonb)
+}
+
+/// Insert multiple JSON values for term filter search testing.
+/// Creates rows with mixed case strings that should match when queried with lowercase.
+pub async fn insert_jsonb_filtered_for_search() -> Vec<(i64, Value)> {
+    let test_data = vec![
+        serde_json::json!({"name": "Alice", "number": 1}),
+        serde_json::json!({"name": "BOB", "number": 2}),
+        serde_json::json!({"name": "Charlie", "number": 3}),
+        serde_json::json!({"name": "DIANA", "number": 4}),
+        serde_json::json!({"name": "Eve", "number": 5}),
+    ];
+
+    let mut results = Vec::new();
+
+    for encrypted_jsonb in test_data {
+        let id = random_id();
+
+        let sql = "INSERT INTO encrypted (id, encrypted_jsonb_filtered) VALUES ($1, $2)";
+        insert(sql, &[&id, &encrypted_jsonb]).await;
+
+        // Verify encryption actually occurred
+        assert_encrypted_jsonb_filtered(id, &encrypted_jsonb).await;
+
+        results.push((id, encrypted_jsonb));
+    }
+
+    results
+}
+
+/// Verifies that a JSON value in encrypted_jsonb_filtered was actually encrypted.
+pub async fn assert_encrypted_jsonb_filtered(id: i64, plaintext: &Value) {
+    let sql = "SELECT encrypted_jsonb_filtered::text FROM encrypted WHERE id = $1";
+    let stored: Vec<String> = query_direct_by(sql, &id).await;
+
+    assert_eq!(stored.len(), 1, "Expected exactly one row");
+    let stored_text = &stored[0];
+
+    let plaintext_str = plaintext.to_string();
+    assert_ne!(
+        stored_text, &plaintext_str,
+        "ENCRYPTION FAILED for encrypted_jsonb_filtered: Stored value matches plaintext! Data was not encrypted."
+    );
+
+    // Additional verification: the encrypted format should be different structure
+    if let Ok(stored_json) = serde_json::from_str::<Value>(stored_text) {
+        assert_ne!(
+            stored_json, *plaintext,
+            "ENCRYPTION FAILED for encrypted_jsonb_filtered: Stored JSON structure matches plaintext!"
+        );
+    }
+}
+
 /// Verifies that a numeric value was actually encrypted in the database.
 /// Queries directly (bypassing proxy) and asserts stored value differs from plaintext.
 pub async fn assert_encrypted_numeric<T>(id: i64, column: &str, plaintext: T)

packages/cipherstash-proxy-integration/src/select/jsonb_term_filter.rs

@@ -0,0 +1,155 @@
+//! Tests for term filters on SteVec indexes.
+//!
+//! The `encrypted_jsonb_filtered` column has a downcase term filter configured,
+//! meaning all string values are lowercased before encryption. This enables
+//! case-insensitive queries - but note that the decrypted data is also lowercased.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{
+        clear, insert_jsonb_filtered, insert_jsonb_filtered_for_search, query_by_params,
+        simple_query, trace,
+    };
+    use crate::support::json_path::JsonPath;
+    use serde_json::Value;
+
+    /// Test case-insensitive equality matching with the downcase term filter.
+    /// Data is inserted with mixed case ("Alice", "BOB") but stored/returned as lowercase.
+    #[tokio::test]
+    async fn select_jsonb_filtered_case_insensitive_eq() {
+        trace();
+        clear().await;
+        insert_jsonb_filtered_for_search().await;
+
+        // Query with lowercase "alice" should match the row originally inserted as "Alice"
+        let selector = "name";
+        let value = Value::from("alice");
+
+        // Extended protocol
+        let sql =
+            "SELECT encrypted_jsonb_filtered FROM encrypted WHERE encrypted_jsonb_filtered -> $1 = $2";
+        let actual = query_by_params::<Value>(sql, &[&selector, &value]).await;
+
+        // Term filter lowercases during encryption, so returned value is lowercase
+        assert_eq!(actual.len(), 1);
+        assert_eq!(actual[0]["name"], "alice");
+        assert_eq!(actual[0]["number"], 1);
+    }
+
+    /// Test that data inserted with uppercase is stored and returned as lowercase
+    #[tokio::test]
+    async fn select_jsonb_filtered_uppercase_query_matches() {
+        trace();
+        clear().await;
+        insert_jsonb_filtered_for_search().await;
+
+        // Query with "bob" should match the row originally inserted as "BOB"
+        let selector = "name";
+        let value = Value::from("bob");
+
+        let sql =
+            "SELECT encrypted_jsonb_filtered FROM encrypted WHERE encrypted_jsonb_filtered -> $1 = $2";
+        let actual = query_by_params::<Value>(sql, &[&selector, &value]).await;
+
+        // Both stored and queried values are lowercased
+        assert_eq!(actual.len(), 1);
+        assert_eq!(actual[0]["name"], "bob");
+        assert_eq!(actual[0]["number"], 2);
+    }
+
+    /// Test simple protocol with case-insensitive matching
+    #[tokio::test]
+    async fn select_jsonb_filtered_simple_protocol() {
+        trace();
+        clear().await;
+        insert_jsonb_filtered_for_search().await;
+
+        // Simple protocol query - value is lowercased on both sides
+        let sql =
+            "SELECT encrypted_jsonb_filtered FROM encrypted WHERE encrypted_jsonb_filtered -> 'name' = '\"charlie\"'";
+        let actual = simple_query::<Value>(sql).await;
+
+        assert_eq!(actual.len(), 1);
+        assert_eq!(actual[0]["name"], "charlie");
+        assert_eq!(actual[0]["number"], 3);
+    }
+
+    /// Test that numbers are not affected by the downcase filter
+    #[tokio::test]
+    async fn select_jsonb_filtered_numbers_unchanged() {
+        trace();
+        clear().await;
+        insert_jsonb_filtered_for_search().await;
+
+        let selector = "number";
+        let value = Value::from(4);
+
+        let sql =
+            "SELECT encrypted_jsonb_filtered FROM encrypted WHERE encrypted_jsonb_filtered -> $1 = $2";
+        let actual = query_by_params::<Value>(sql, &[&selector, &value]).await;
+
+        assert_eq!(actual.len(), 1);
+        // Name is lowercased by term filter
+        assert_eq!(actual[0]["name"], "diana");
+        assert_eq!(actual[0]["number"], 4);
+    }
+
+    /// Test case-insensitive matching using jsonb_path_query_first
+    #[tokio::test]
+    async fn select_jsonb_filtered_path_query_case_insensitive() {
+        trace();
+        clear().await;
+        insert_jsonb_filtered_for_search().await;
+
+        let json_path_selector = JsonPath::new("name");
+        let value = Value::from("eve");
+
+        let sql =
+            "SELECT encrypted_jsonb_filtered FROM encrypted WHERE jsonb_path_query_first(encrypted_jsonb_filtered, $1) = $2";
+        let actual = query_by_params::<Value>(sql, &[&json_path_selector, &value]).await;
+
+        assert_eq!(actual.len(), 1);
+        assert_eq!(actual[0]["name"], "eve");
+        assert_eq!(actual[0]["number"], 5);
+    }
+
+    /// Test nested field access with term filter
+    #[tokio::test]
+    async fn select_jsonb_filtered_nested_case_insensitive() {
+        trace();
+        clear().await;
+        let (_id, _) = insert_jsonb_filtered().await;
+
+        // The fixture has nested.title = "Engineer" which gets lowercased
+        // Query with lowercase should match
+        let json_path_selector = JsonPath::new("nested.title");
+        let value = Value::from("engineer");
+
+        let sql =
+            "SELECT encrypted_jsonb_filtered FROM encrypted WHERE jsonb_path_query_first(encrypted_jsonb_filtered, $1) = $2";
+        let actual = query_by_params::<Value>(sql, &[&json_path_selector, &value]).await;
+
+        assert_eq!(actual.len(), 1);
+        assert_eq!(actual[0]["nested"]["title"], "engineer");
+    }
+
+    /// Test that original fixture data is correctly inserted and queryable
+    #[tokio::test]
+    async fn select_jsonb_filtered_fixture_data() {
+        trace();
+        clear().await;
+        let (_id, _expected) = insert_jsonb_filtered().await;
+
+        // Query by name field - both query and stored data are lowercased
+        let selector = "name";
+        let value = Value::from("john");
+
+        let sql =
+            "SELECT encrypted_jsonb_filtered FROM encrypted WHERE encrypted_jsonb_filtered -> $1 = $2";
+        let actual = query_by_params::<Value>(sql, &[&selector, &value]).await;
+
+        assert_eq!(actual.len(), 1);
+        assert_eq!(actual[0]["name"], "john");
+        assert_eq!(actual[0]["city"], "melbourne");
+    }
+}

packages/cipherstash-proxy-integration/src/select/mod.rs

@@ -9,6 +9,7 @@ mod jsonb_get_field_as_ciphertext;
 mod jsonb_path_exists;
 mod jsonb_path_query;
 mod jsonb_path_query_first;
+mod jsonb_term_filter;
 mod order_by;
 mod order_by_with_null;
 mod pg_catalog;

packages/cipherstash-proxy/src/proxy/encrypt_config/config.rs

@@ -96,6 +96,8 @@ pub struct MatchIndexOpts {
 #[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
 pub struct SteVecIndexOpts {
     prefix: String,
+    #[serde(default)]
+    term_filters: Vec<TokenFilter>,
 }
 
 fn default_tokenizer() -> Tokenizer {
@@ -182,10 +184,14 @@ impl Column {
             }))
         }
 
-        if let Some(SteVecIndexOpts { prefix }) = self.indexes.ste_vec_index {
+        if let Some(SteVecIndexOpts {
+            prefix,
+            term_filters,
+        }) = self.indexes.ste_vec_index
+        {
             config = config.add_index(Index::new(IndexType::SteVec {
                 prefix,
-                term_filters: vec![],
+                term_filters,
             }))
         }
 

tests/sql/schema.sql

@@ -33,6 +33,7 @@ CREATE TABLE encrypted (
     encrypted_float8 eql_v2_encrypted,
     encrypted_date eql_v2_encrypted,
     encrypted_jsonb eql_v2_encrypted,
+    encrypted_jsonb_filtered eql_v2_encrypted,
     PRIMARY KEY(id)
 );
 
@@ -157,6 +158,14 @@ SELECT eql_v2.add_search_config(
   '{"prefix": "encrypted/encrypted_jsonb"}'
 );
 
+SELECT eql_v2.add_search_config(
+  'encrypted',
+  'encrypted_jsonb_filtered',
+  'ste_vec',
+  'jsonb',
+  '{"prefix": "encrypted/encrypted_jsonb_filtered", "term_filters": [{"kind": "downcase"}]}'
+);
+
 SELECT eql_v2.add_encrypted_constraint('encrypted', 'encrypted_text');
 
 
@@ -177,6 +186,7 @@ CREATE TABLE encrypted_elixir (
     encrypted_float8 eql_v2_encrypted,
     encrypted_date eql_v2_encrypted,
     encrypted_jsonb eql_v2_encrypted,
+    encrypted_jsonb_filtered eql_v2_encrypted,
     PRIMARY KEY(id)
 );
 
@@ -301,5 +311,13 @@ SELECT eql_v2.add_search_config(
   '{"prefix": "encrypted/encrypted_jsonb"}'
 );
 
+SELECT eql_v2.add_search_config(
+  'encrypted_elixir',
+  'encrypted_jsonb_filtered',
+  'ste_vec',
+  'jsonb',
+  '{"prefix": "encrypted/encrypted_jsonb_filtered", "term_filters": [{"kind": "downcase"}]}'
+);
+
 SELECT eql_v2.add_encrypted_constraint('encrypted_elixir', 'encrypted_text');