From cc3f9bff160a6139cbfb136b8a8c1b4c6cfff690 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 5 May 2025 17:38:03 +1000 Subject: [PATCH 1/9] chore(eql): Update test schema to use EQL 2.0 --- tests/sql/schema.sql | 54 ++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/sql/schema.sql b/tests/sql/schema.sql index c1398811..32198f54 100644 --- a/tests/sql/schema.sql +++ b/tests/sql/schema.sql @@ -13,95 +13,95 @@ DROP TABLE IF EXISTS encrypted; CREATE TABLE encrypted ( id bigint, plaintext text, - encrypted_text cs_encrypted_v1, - encrypted_bool cs_encrypted_v1, - encrypted_int2 cs_encrypted_v1, - encrypted_int4 cs_encrypted_v1, - encrypted_int8 cs_encrypted_v1, - encrypted_float8 cs_encrypted_v1, - encrypted_date cs_encrypted_v1, - encrypted_jsonb cs_encrypted_v1, + encrypted_text eql_v1_encrypted, + encrypted_bool eql_v1_encrypted, + encrypted_int2 eql_v1_encrypted, + encrypted_int4 eql_v1_encrypted, + encrypted_int8 eql_v1_encrypted, + encrypted_float8 eql_v1_encrypted, + encrypted_date eql_v1_encrypted, + encrypted_jsonb eql_v1_encrypted, PRIMARY KEY(id) ); DROP TABLE IF EXISTS unconfigured; CREATE TABLE unconfigured ( id bigint, - encrypted_unconfigured cs_encrypted_v1, + encrypted_unconfigured eql_v1_encrypted, PRIMARY KEY(id) ); -SELECT cs_add_index_v1( +SELECT eql_v1.add_index( 'encrypted', 'encrypted_text', 'unique', 'text' ); -SELECT cs_add_index_v1( +SELECT eql_v1.add_index( 'encrypted', 'encrypted_text', 'match', 'text' ); -SELECT cs_add_index_v1( +SELECT eql_v1.add_index( 'encrypted', 'encrypted_text', 'ore', 'text' ); -SELECT cs_add_index_v1( +SELECT eql_v1.add_index( 'encrypted', 'encrypted_bool', 'unique', 'boolean' ); -SELECT cs_add_index_v1( +SELECT eql_v1.add_index( 'encrypted', 'encrypted_bool', 'ore', 'boolean' ); -SELECT cs_add_index_v1( +SELECT eql_v1.add_index( 'encrypted', 'encrypted_int2', 'unique', 'small_int' ); -SELECT cs_add_index_v1( +SELECT eql_v1.add_index( 'encrypted', 'encrypted_int2', 'ore', 'small_int' ); -SELECT cs_add_index_v1( +SELECT eql_v1.add_index( 'encrypted', 'encrypted_int4', 'unique', 'int' ); -SELECT cs_add_index_v1( +SELECT eql_v1.add_index( 'encrypted', 'encrypted_int4', 'ore', 'int' ); -SELECT cs_add_index_v1( +SELECT eql_v1.add_index( 'encrypted', 'encrypted_int8', 'unique', 'big_int' ); -SELECT cs_add_index_v1( +SELECT eql_v1.add_index( 'encrypted', 'encrypted_int8', 'ore', @@ -109,35 +109,35 @@ SELECT cs_add_index_v1( ); -SELECT cs_add_index_v1( +SELECT eql_v1.add_index( 'encrypted', 'encrypted_float8', 'unique', 'double' ); -SELECT cs_add_index_v1( +SELECT eql_v1.add_index( 'encrypted', 'encrypted_float8', 'ore', 'double' ); -SELECT cs_add_index_v1( +SELECT eql_v1.add_index( 'encrypted', 'encrypted_date', 'unique', 'date' ); -SELECT cs_add_index_v1( +SELECT eql_v1.add_index( 'encrypted', 'encrypted_date', 'ore', 'date' ); -SELECT cs_add_index_v1( +SELECT eql_v1.add_index( 'encrypted', 'encrypted_jsonb', 'ste_vec', @@ -145,5 +145,5 @@ SELECT cs_add_index_v1( '{"prefix": "encrypted/encrypted_jsonb"}' ); -SELECT cs_encrypt_v1(); -SELECT cs_activate_v1(); +SELECT eql_v1.encrypt(); +SELECT eql_v1.activate(); From e635745bced8418318506c1fef3f5721448cc38f Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 6 May 2025 09:10:26 +1000 Subject: [PATCH 2/9] chore(eql): Update to use EQL 2.0 configuration table --- docs/getting-started/schema-example.sql | 2 +- packages/cipherstash-proxy/src/encrypt/config/manager.rs | 3 +-- .../cipherstash-proxy/src/encrypt/sql/select_config.sql | 2 +- tests/benchmark/sql/benchmark-schema.sql | 2 +- tests/sql/schema-uninstall.sql | 2 +- tests/sql/schema.sql | 2 +- tests/tasks/test/integration/psql-passthrough.sh | 6 +++--- 7 files changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/getting-started/schema-example.sql b/docs/getting-started/schema-example.sql index 29e3e743..c6ad0cb0 100644 --- a/docs/getting-started/schema-example.sql +++ b/docs/getting-started/schema-example.sql @@ -1,4 +1,4 @@ -TRUNCATE TABLE cs_configuration_v1; +TRUNCATE TABLE public.eql_v1_configuration; -- Exciting cipherstash table DROP TABLE IF EXISTS users; diff --git a/packages/cipherstash-proxy/src/encrypt/config/manager.rs b/packages/cipherstash-proxy/src/encrypt/config/manager.rs index 31312533..df23d8ab 100644 --- a/packages/cipherstash-proxy/src/encrypt/config/manager.rs +++ b/packages/cipherstash-proxy/src/encrypt/config/manager.rs @@ -195,8 +195,7 @@ pub async fn load_encrypt_config(config: &DatabaseConfig) -> Result bool { let msg = e.to_string(); - msg.contains("cs_configuration_v1") && msg.contains("does not exist") + msg.contains("eql_v1_configuration") && msg.contains("does not exist") } diff --git a/packages/cipherstash-proxy/src/encrypt/sql/select_config.sql b/packages/cipherstash-proxy/src/encrypt/sql/select_config.sql index 72827f37..8be0732f 100644 --- a/packages/cipherstash-proxy/src/encrypt/sql/select_config.sql +++ b/packages/cipherstash-proxy/src/encrypt/sql/select_config.sql @@ -1 +1 @@ -SELECT data FROM cs_configuration_v1 WHERE state = 'active' LIMIT 1; +SELECT data FROM public.eql_v1_configuration WHERE state = 'active' LIMIT 1; diff --git a/tests/benchmark/sql/benchmark-schema.sql b/tests/benchmark/sql/benchmark-schema.sql index fecdf7bf..e975f2e8 100644 --- a/tests/benchmark/sql/benchmark-schema.sql +++ b/tests/benchmark/sql/benchmark-schema.sql @@ -1,4 +1,4 @@ -TRUNCATE TABLE cs_configuration_v1; +TRUNCATE TABLE public.eql_v1_configuration; DROP TABLE IF EXISTS benchmark_plaintext; CREATE TABLE benchmark_plaintext ( diff --git a/tests/sql/schema-uninstall.sql b/tests/sql/schema-uninstall.sql index 3c34ba76..ae6630cf 100644 --- a/tests/sql/schema-uninstall.sql +++ b/tests/sql/schema-uninstall.sql @@ -1,4 +1,4 @@ -DROP TABLE IF EXISTS cs_configuration_v1; +DROP TABLE IF EXISTS public.eql_v1_configuration; -- Regular old table DROP TABLE IF EXISTS plaintext; diff --git a/tests/sql/schema.sql b/tests/sql/schema.sql index 32198f54..57f5e29f 100644 --- a/tests/sql/schema.sql +++ b/tests/sql/schema.sql @@ -1,4 +1,4 @@ -TRUNCATE TABLE cs_configuration_v1; +TRUNCATE TABLE public.eql_v1_configuration; -- Regular old table DROP TABLE IF EXISTS plaintext; diff --git a/tests/tasks/test/integration/psql-passthrough.sh b/tests/tasks/test/integration/psql-passthrough.sh index e07fd77d..c9319539 100755 --- a/tests/tasks/test/integration/psql-passthrough.sh +++ b/tests/tasks/test/integration/psql-passthrough.sh @@ -17,10 +17,10 @@ EOF # Confirm that there is indeed no config set +e -OUTPUT="$(docker exec -i postgres${CONTAINER_SUFFIX} psql 'postgresql://cipherstash:password@proxy:6432/cipherstash?sslmode=disable' --command 'SELECT * FROM cs_configuration_v1' 2>&1)" +OUTPUT="$(docker exec -i postgres${CONTAINER_SUFFIX} psql 'postgresql://cipherstash:password@proxy:6432/cipherstash?sslmode=disable' --command 'SELECT * FROM eql_v1_configuration' 2>&1)" retval=$? -if echo ${OUTPUT} | grep -v 'relation "cs_configuration_v1" does not exist'; then - echo "error: did not see string in output: \"relation "cs_configuration_v1" does not exist\"" +if echo ${OUTPUT} | grep -v 'relation "eql_v1_configuration" does not exist'; then + echo "error: did not see string in output: \"relation "eql_v1_configuration" does not exist\"" exit 1 fi From 8038627e5bf13ff51292ebcba9745b9b33354eea Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 6 May 2025 10:48:36 +1000 Subject: [PATCH 3/9] fix: Uninstall should use CS_EQL_UNINSTALL_PATH if defined --- mise.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mise.toml b/mise.toml index 9cf04e2a..db0a71cb 100644 --- a/mise.toml +++ b/mise.toml @@ -421,7 +421,7 @@ cat sql/schema.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql: """ [tasks."postgres:eql:teardown"] -alias = 's' +alias = 'teardown' description = "Uninstalls EQL and removes schema from database" run = """ #!/bin/bash @@ -512,7 +512,7 @@ fi if [ -z "$CS_EQL_UNINSTALL_PATH" ]; then curl -sLo sql/cipherstash-encrypt-uninstall.sql https://github.com/cipherstash/encrypt-query-language/releases/download/${CS_EQL_VERSION}/cipherstash-encrypt-uninstall.sql else - echo "Using EQL: ${CS_EQL_PATH}" + echo "Using EQL: ${CS_EQL_UNINSTALL_PATH}" cp "$CS_EQL_UNINSTALL_PATH" sql/cipherstash-encrypt-uninstall.sql fi """ From e06e7513a0b0e83d895114c357e340db6ae31451 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 6 May 2025 14:48:12 +1000 Subject: [PATCH 4/9] chore: make postgres:setup depend on postgres:teardown --- mise.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mise.toml b/mise.toml index db0a71cb..cef234e3 100644 --- a/mise.toml +++ b/mise.toml @@ -410,6 +410,7 @@ fi [tasks."postgres:setup"] alias = 's' +depends = ["postgres:eql:teardown"] description = "Installs EQL and applies schema to database" run = """ #!/bin/bash @@ -421,7 +422,6 @@ cat sql/schema.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql: """ [tasks."postgres:eql:teardown"] -alias = 'teardown' description = "Uninstalls EQL and removes schema from database" run = """ #!/bin/bash From 3874bbf6c99d81ecc5eaee210923a20f667cd5e3 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 6 May 2025 16:18:21 +1000 Subject: [PATCH 5/9] fix: Use updated eql_v1_encrypted column type --- docs/errors.md | 4 ++-- docs/getting-started/schema-example.sql | 6 +++--- docs/how-to.md | 8 ++++---- mise.toml | 12 +----------- .../src/extended_protocol_error_messages.rs | 4 ++-- .../cipherstash-proxy/src/encrypt/schema/manager.rs | 4 ++-- .../src/postgresql/messages/parse.rs | 8 ++++---- 7 files changed, 18 insertions(+), 28 deletions(-) diff --git a/docs/errors.md b/docs/errors.md index 7e5faceb..1105d064 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -314,7 +314,7 @@ For example: ## Unknown Column -The column has an encrypted type (PostgreSQL `cs_encrypted_v1` type ) with no encryption configuration. +The column has an encrypted type (PostgreSQL `eql_v1_encrypted` type ) with no encryption configuration. Without the configuration, Cipherstash Proxy does not know how to encrypt the column. Any data is unprotected and unencrypted. @@ -341,7 +341,7 @@ Column 'column_name' in table 'table_name' has no Encrypt configuration ## Unknown Table -The table has one or more encrypted columns (PostgreSQL `cs_encrypted_v1` type ) with no encryption configuration. +The table has one or more encrypted columns (PostgreSQL `eql_v1_encrypted` type ) with no encryption configuration. Without the configuration, Cipherstash Proxy does not know how to encrypt the column. Any data is unprotected and unencrypted. diff --git a/docs/getting-started/schema-example.sql b/docs/getting-started/schema-example.sql index c6ad0cb0..0120cde4 100644 --- a/docs/getting-started/schema-example.sql +++ b/docs/getting-started/schema-example.sql @@ -4,9 +4,9 @@ TRUNCATE TABLE public.eql_v1_configuration; DROP TABLE IF EXISTS users; CREATE TABLE users ( id SERIAL PRIMARY KEY, - encrypted_email cs_encrypted_v1, - encrypted_dob cs_encrypted_v1, - encrypted_salary cs_encrypted_v1 + encrypted_email eql_v1_encrypted, + encrypted_dob eql_v1_encrypted, + encrypted_salary eql_v1_encrypted ); SELECT cs_add_index_v1( diff --git a/docs/how-to.md b/docs/how-to.md index a38f80cb..cac141c0 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -162,22 +162,22 @@ This will output the version of EQL installed. In your existing PostgreSQL database, you store your data in tables and columns. Those columns have types like `integer`, `text`, `timestamp`, and `boolean`. -When storing encrypted data in PostgreSQL with Proxy, you use a special column type called `cs_encrypted_v1`, which is [provided by EQL](#setting-up-the-database-schema). -`cs_encrypted_v1` is a container column type that can be used for any type of encrypted data you want to store or search, whether they are numbers (`int`, `small_int`, `big_int`), text (`text`), dates and times (`date`), or booleans (`boolean`). +When storing encrypted data in PostgreSQL with Proxy, you use a special column type called `eql_v1_encrypted`, which is [provided by EQL](#setting-up-the-database-schema). +`eql_v1_encrypted` is a container column type that can be used for any type of encrypted data you want to store or search, whether they are numbers (`int`, `small_int`, `big_int`), text (`text`), dates and times (`date`), or booleans (`boolean`). Create a table with an encrypted column for `email`: ```sql CREATE TABLE users ( id SERIAL PRIMARY KEY, - email cs_encrypted_v1 + email eql_v1_encrypted ) ``` This creates a `users` table with two columns: - `id`, an autoincrementing integer column that is the primary key for the record - - `email`, a `cs_encrypted_v1` column + - `email`, a `eql_v1_encrypted` column There are important differences between the plaintext columns you've traditionally used in PostgreSQL and encrypted columns with CipherStash Proxy: diff --git a/mise.toml b/mise.toml index cef234e3..01769875 100644 --- a/mise.toml +++ b/mise.toml @@ -567,7 +567,7 @@ cp -v {{config_root}}/target/{{ target }}/release/cipherstash-proxy {{config_roo """ [tasks."build:docker"] -depends = ["build:docker:fetch_eql"] +depends = ["postgres:eql:download"] description = "Build a Docker image for cipherstash-proxy" run = """ {% set default_platform = "linux/" ~ arch() | replace(from="x86_64", to="amd64") %} @@ -580,16 +580,6 @@ docker build . \ --platform {{option(name="platform",default=default_platform)}} \ """ -[tasks."build:docker:fetch_eql"] -description = "Fetch the EQL installation script" -run = """ -if [ ! -e "cipherstash-eql.sql" ]; then - echo "Fetching: cipherstash-eql.sql" - curl -sLo cipherstash-eql.sql https://github.com/cipherstash/encrypt-query-language/releases/download/${CS_EQL_VERSION}/cipherstash-encrypt.sql -else - echo "Prefetched: cipherstash-eql.sql" -fi -""" [tasks.release] description = "Publish release artifacts" diff --git a/packages/cipherstash-proxy-integration/src/extended_protocol_error_messages.rs b/packages/cipherstash-proxy-integration/src/extended_protocol_error_messages.rs index c45ec4af..37ac99ea 100644 --- a/packages/cipherstash-proxy-integration/src/extended_protocol_error_messages.rs +++ b/packages/cipherstash-proxy-integration/src/extended_protocol_error_messages.rs @@ -67,10 +67,10 @@ mod tests { let msg = err.to_string(); // This is similar to below. The error message comes from tokio-postgres when Proxy - // returns cs_encrypted_v1 and the client cannot convert to a string. + // returns eql_v1_encrypted and the client cannot convert to a string. // If mapping errors are enabled (enable_mapping_errors or CS_DEVELOPMENT__ENABLE_MAPPING_ERRORS), // then Proxy will return an error that says "Column X in table Y has no Encrypt configuration" - assert_eq!(msg, "error serializing parameter 1: cannot convert between the Rust type `&str` and the Postgres type `cs_encrypted_v1`"); + assert_eq!(msg, "error serializing parameter 1: cannot convert between the Rust type `&str` and the Postgres type `eql_v1_encrypted`"); } else { unreachable!(); } diff --git a/packages/cipherstash-proxy/src/encrypt/schema/manager.rs b/packages/cipherstash-proxy/src/encrypt/schema/manager.rs index fc000ccb..f005a16f 100644 --- a/packages/cipherstash-proxy/src/encrypt/schema/manager.rs +++ b/packages/cipherstash-proxy/src/encrypt/schema/manager.rs @@ -143,8 +143,8 @@ pub async fn load_schema(config: &DatabaseConfig) -> Result { let ident = Ident::with_quote('"', col); let column = match domain.as_deref() { - Some("cs_encrypted_v1") => { - debug!(target: SCHEMA, msg = "cs_encrypted_v1 column", table = table_name, column = col); + Some("eql_v1_encrypted") => { + debug!(target: SCHEMA, msg = "eql_v1_encrypted column", table = table_name, column = col); Column::eql(ident) } _ => Column::native(ident), diff --git a/packages/cipherstash-proxy/src/postgresql/messages/parse.rs b/packages/cipherstash-proxy/src/postgresql/messages/parse.rs index faf0ec6b..a2c30f5c 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/parse.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/parse.rs @@ -24,11 +24,11 @@ impl Parse { } /// - /// Encrypted columns are the cs_encrypted_v1 Domain Type - /// cs_encrypted_v1 wraps JSONB + /// Encrypted columns are the eql_v1_encrypted Domain Type + /// eql_v1_encrypted wraps JSONB /// - /// Using JSONB to avoid the complexity of loading the OID of cs_encrypted_v1 - /// PostgreSQL will coerce JSONB to cs_encrypted_v1 if it passes the constaint check + /// Using JSONB to avoid the complexity of loading the OID of eql_v1_encrypted + /// PostgreSQL will coerce JSONB to eql_v1_encrypted if it passes the constaint check /// pub fn rewrite_param_types(&mut self, columns: &[Option]) { for (idx, col) in columns.iter().enumerate() { From d936353bf793ee17bc2e468d22a83f2004dcabfa Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 6 May 2025 16:38:39 +1000 Subject: [PATCH 6/9] chore: update schema load to check for new type --- packages/cipherstash-proxy/src/encrypt/schema/manager.rs | 7 +++---- .../src/encrypt/sql/select_table_schemas.sql | 6 ++++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/cipherstash-proxy/src/encrypt/schema/manager.rs b/packages/cipherstash-proxy/src/encrypt/schema/manager.rs index f005a16f..b2603a98 100644 --- a/packages/cipherstash-proxy/src/encrypt/schema/manager.rs +++ b/packages/cipherstash-proxy/src/encrypt/schema/manager.rs @@ -132,17 +132,16 @@ pub async fn load_schema(config: &DatabaseConfig) -> Result { let table_name: String = table.get("table_name"); let primary_keys: Vec> = table.get("primary_keys"); let columns: Vec = table.get("columns"); - let _types: Vec> = table.get("column_types"); - let domains: Vec> = table.get("column_domains"); + let column_type_names: Vec> = table.get("column_type_names"); let mut table = Table::new(Ident::new(&table_name)); - columns.iter().zip(domains).for_each(|(col, domain)| { + columns.iter().zip(column_type_names).for_each(|(col, column_type_name)| { let is_primary_key = primary_keys.contains(&Some(col.to_string())); let ident = Ident::with_quote('"', col); - let column = match domain.as_deref() { + let column = match column_type_name.as_deref() { Some("eql_v1_encrypted") => { debug!(target: SCHEMA, msg = "eql_v1_encrypted column", table = table_name, column = col); Column::eql(ident) diff --git a/packages/cipherstash-proxy/src/encrypt/sql/select_table_schemas.sql b/packages/cipherstash-proxy/src/encrypt/sql/select_table_schemas.sql index ee3ba513..88743f3e 100644 --- a/packages/cipherstash-proxy/src/encrypt/sql/select_table_schemas.sql +++ b/packages/cipherstash-proxy/src/encrypt/sql/select_table_schemas.sql @@ -3,8 +3,7 @@ SELECT t.table_name, array_agg(distinct k.column_name)::text[] AS primary_keys, array_agg(c.column_name)::text[] AS columns, - array_agg(c.data_type)::text[] AS column_types, - array_agg(c.domain_name)::text[] AS column_domains + array_agg(c.udt_name)::text[] AS column_type_names FROM information_schema.tables t LEFT JOIN @@ -24,3 +23,6 @@ GROUP BY t.table_schema, t.table_name ORDER BY t.table_schema, t.table_name; + + + From 2dec80adc4c26553d4c15d70715689452a9b468b Mon Sep 17 00:00:00 2001 From: James Sadler Date: Tue, 6 May 2025 16:35:53 +1000 Subject: [PATCH 7/9] chore(mapper): replaced literals are now `ROW(..)` expressions --- packages/eql-mapper/src/lib.rs | 58 +++++++++++++++---- .../replace_plaintext_eql_literals.rs | 35 +++++++++-- 2 files changed, 77 insertions(+), 16 deletions(-) diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index 3bd45186..3d72df77 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -1017,27 +1017,63 @@ mod test { )] ); - let transformed_statement = match typed.transform(HashMap::from_iter([( + match typed.transform(HashMap::from_iter([( typed.literals[0].1.as_node_key(), ast::Value::SingleQuotedString("ENCRYPTED".into()), )])) { - Ok(transformed_statement) => transformed_statement, + Ok(transformed_statement) => assert_eq!( + transformed_statement.to_string(), + "SELECT * FROM employees WHERE salary > ROW('ENCRYPTED'::JSONB)" + ), Err(err) => panic!("statement transformation failed: {}", err), }; + } + + #[test] + fn insert_with_literal_subsitution() { + // init_tracing(); + + let schema = resolver(schema! { + tables: { + employees: { + id, + salary (EQL), + } + } + }); + + let statement = parse( + r#" + insert into employees (salary) values (20000) + "#, + ); - // This type checks the transformed statement so we can get hold of the encrypted literal. - let typed = match type_check(schema, &transformed_statement) { + let typed = match type_check(schema.clone(), &statement) { Ok(typed) => typed, Err(err) => panic!("type check failed: {:#?}", err), }; - assert!(typed.literals.contains(&( - EqlValue(TableColumn { - table: id("employees"), - column: id("salary") - }), - &ast::Value::SingleQuotedString("ENCRYPTED".into()), - ))); + assert_eq!( + typed.literals, + vec![( + EqlValue(TableColumn { + table: id("employees"), + column: id("salary") + }), + &ast::Value::Number(20000.into(), false) + )] + ); + + match typed.transform(HashMap::from_iter([( + typed.literals[0].1.as_node_key(), + ast::Value::SingleQuotedString("ENCRYPTED".into()), + )])) { + Ok(transformed_statement) => assert_eq!( + transformed_statement.to_string(), + "INSERT INTO employees (salary) VALUES (ROW('ENCRYPTED'::JSONB))" + ), + Err(err) => panic!("statement transformation failed: {}", err), + }; } #[test] diff --git a/packages/eql-mapper/src/transformation_rules/replace_plaintext_eql_literals.rs b/packages/eql-mapper/src/transformation_rules/replace_plaintext_eql_literals.rs index 687649af..a153b115 100644 --- a/packages/eql-mapper/src/transformation_rules/replace_plaintext_eql_literals.rs +++ b/packages/eql-mapper/src/transformation_rules/replace_plaintext_eql_literals.rs @@ -1,6 +1,9 @@ use std::{any::type_name, collections::HashMap}; -use sqltk::parser::ast::Value; +use sqltk::parser::ast::{ + CastKind, DataType, Expr, Function, FunctionArg, FunctionArgExpr, FunctionArgumentList, + FunctionArguments, Ident, ObjectName, Value, +}; use sqltk::{NodeKey, NodePath, Visitable}; use crate::EqlMapperError; @@ -25,10 +28,10 @@ impl<'ast> TransformationRule<'ast> for ReplacePlaintextEqlLiterals<'ast> { target_node: &mut N, ) -> Result { if self.would_edit(node_path, target_node) { - if let Some((value,)) = node_path.last_1_as::() { + if let Some((Expr::Value(value),)) = node_path.last_1_as::() { if let Some(replacement) = self.encrypted_literals.remove(&NodeKey::new(value)) { - let target_node = target_node.downcast_mut::().unwrap(); - *target_node = replacement; + let target_node = target_node.downcast_mut::().unwrap(); + *target_node = make_row_expression(replacement); return Ok(true); } } @@ -38,7 +41,7 @@ impl<'ast> TransformationRule<'ast> for ReplacePlaintextEqlLiterals<'ast> { } fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { - if let Some((value,)) = node_path.last_1_as::() { + if let Some((Expr::Value(value),)) = node_path.last_1_as::() { return self.encrypted_literals.contains_key(&NodeKey::new(value)); } false @@ -55,3 +58,25 @@ impl<'ast> TransformationRule<'ast> for ReplacePlaintextEqlLiterals<'ast> { } } } + +fn make_row_expression(replacement: Value) -> Expr { + Expr::Function(Function { + name: ObjectName(vec![Ident::new("ROW")]), + uses_odbc_syntax: false, + parameters: FunctionArguments::None, + args: FunctionArguments::List(FunctionArgumentList { + duplicate_treatment: None, + clauses: vec![], + args: vec![FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Cast { + kind: CastKind::DoubleColon, + expr: Box::new(Expr::Value(replacement)), + data_type: DataType::JSONB, + format: None, + }))], + }), + filter: None, + null_treatment: None, + over: None, + within_group: vec![], + }) +} From 098f1ccb7f846249b72d0d6dffb7cc55303fbd54 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Tue, 6 May 2025 16:54:59 +1000 Subject: [PATCH 8/9] =?UTF-8?q?refactor(mise):=20unify=20EQL=20download=20?= =?UTF-8?q?tasks=20to=20One=20True=20Way=E2=84=A2=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 5 ++--- mise.toml | 47 ++++++++++++++++++++++++++++------------------- proxy.Dockerfile | 2 +- 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index a5b17a57..05413b30 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,6 @@ /cipherstash-proxy.local.toml mise.local.toml tests/pg/data** -tests/sql/cipherstash-encrypt.sql -tests/sql/cipherstash-encrypt-uninstall.sql .vscode rust-toolchain.toml @@ -13,8 +11,9 @@ rust-toolchain.toml # release artifacts /cipherstash-proxy -/cipherstash-eql.sql /packages/cipherstash-proxy/eql-version-at-build-time.txt +/cipherstash-encrypt.sql +/cipherstash-encrypt-uninstall.sql # credentials for local dev .env.proxy.docker diff --git a/mise.toml b/mise.toml index 01769875..64e563df 100644 --- a/mise.toml +++ b/mise.toml @@ -409,27 +409,28 @@ fi """ [tasks."postgres:setup"] -alias = 's' depends = ["postgres:eql:teardown"] +alias = 's' description = "Installs EQL and applies schema to database" run = """ #!/bin/bash cd tests mise run postgres:fail_if_not_running -mise run postgres:eql:download -cat sql/cipherstash-encrypt.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- +cat sql/schema-uninstall.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- +cat ../cipherstash-encrypt-uninstall.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- +cat ../cipherstash-encrypt.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- cat sql/schema.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- """ [tasks."postgres:eql:teardown"] +depends = ["eql:download"] description = "Uninstalls EQL and removes schema from database" run = """ #!/bin/bash cd tests mise run postgres:fail_if_not_running -mise run postgres:eql:download cat sql/schema-uninstall.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- -cat sql/cipherstash-encrypt-uninstall.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- +cat ../cipherstash-encrypt-uninstall.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- """ [tasks."postgres:up"] @@ -490,34 +491,32 @@ for d in tests/pg/data-*; do done """ - -[tasks."postgres:eql:download"] +[tasks."eql:download"] alias = 'e' -description = "Download latest EQL release" +description = "Download latest EQL release or use local copy" dir = "{{config_root}}/tests" outputs = [ - "{{config_root}}/tests/sql/cipherstash-encrypt.sql", - "{{config_root}}/tests/sql/cipherstash-encrypt-uninstall.sql", + "{{config_root}}/cipherstash-encrypt.sql", + "{{config_root}}/cipherstash-encrypt-uninstall.sql", ] run = """ # install script if [ -z "$CS_EQL_PATH" ]; then - curl -sLo sql/cipherstash-encrypt.sql https://github.com/cipherstash/encrypt-query-language/releases/download/${CS_EQL_VERSION}/cipherstash-encrypt.sql + curl -sLo "{{config_root}}/cipherstash-encrypt.sql" https://github.com/cipherstash/encrypt-query-language/releases/download/${CS_EQL_VERSION}/cipherstash-encrypt.sql else - echo "Using EQL: ${CS_EQL_PATH}" - cp "$CS_EQL_PATH" sql/cipherstash-encrypt.sql + echo "Using EQL: ${CS_EQL_PATH}/cipherstash-encrypt.sql" + cp "$CS_EQL_PATH/cipherstash-encrypt.sql" "{{config_root}}/cipherstash-encrypt.sql" fi # uninstall script -if [ -z "$CS_EQL_UNINSTALL_PATH" ]; then - curl -sLo sql/cipherstash-encrypt-uninstall.sql https://github.com/cipherstash/encrypt-query-language/releases/download/${CS_EQL_VERSION}/cipherstash-encrypt-uninstall.sql +if [ -z "$CS_EQL_PATH" ]; then + curl -sLo "{{config_root}}/cipherstash-encrypt-uninstall.sql" https://github.com/cipherstash/encrypt-query-language/releases/download/${CS_EQL_VERSION}/cipherstash-encrypt-uninstall.sql else - echo "Using EQL: ${CS_EQL_UNINSTALL_PATH}" - cp "$CS_EQL_UNINSTALL_PATH" sql/cipherstash-encrypt-uninstall.sql + echo "Using EQL: ${CS_EQL_PATH}/cipherstash-encrypt-uninstall.sql" + cp "$CS_EQL_PATH/cipherstash-encrypt-uninstall.sql" "{{config_root}}/cipherstash-encrypt-uninstall.sql" fi """ - [tasks."python:test"] dir = "{{config_root}}/tests" description = "Runs python tests" @@ -567,7 +566,7 @@ cp -v {{config_root}}/target/{{ target }}/release/cipherstash-proxy {{config_roo """ [tasks."build:docker"] -depends = ["postgres:eql:download"] +depends = ["eql:download"] description = "Build a Docker image for cipherstash-proxy" run = """ {% set default_platform = "linux/" ~ arch() | replace(from="x86_64", to="amd64") %} @@ -580,6 +579,16 @@ docker build . \ --platform {{option(name="platform",default=default_platform)}} \ """ +[tasks."build:docker:fetch_eql"] +description = "Fetch the EQL installation script" +run = """ +if [ ! -e "cipherstash-eql.sql" ]; then + echo "Fetching: cipherstash-eql.sql" + curl -sLo cipherstash-eql.sql https://github.com/cipherstash/encrypt-query-language/releases/download/${CS_EQL_VERSION}/cipherstash-encrypt.sql +else + echo "Prefetched: cipherstash-eql.sql" +fi +""" [tasks.release] description = "Publish release artifacts" diff --git a/proxy.Dockerfile b/proxy.Dockerfile index 27a9d431..02cea0e5 100644 --- a/proxy.Dockerfile +++ b/proxy.Dockerfile @@ -10,7 +10,7 @@ COPY cipherstash-proxy /usr/local/bin/cipherstash-proxy COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh # Copy EQL install scripts -COPY cipherstash-eql.sql /opt/cipherstash-eql.sql +COPY cipherstash-encrypt.sql /opt/cipherstash-eql.sql # Copy example schema COPY docs/getting-started/schema-example.sql /opt/schema-example.sql From b801821e25434123dc0216463118c899ed5c7f4d Mon Sep 17 00:00:00 2001 From: James Sadler Date: Tue, 6 May 2025 16:59:02 +1000 Subject: [PATCH 9/9] fix(proxy): SQL for getting EQL version --- docs/how-to.md | 2 +- packages/cipherstash-proxy/src/encrypt/mod.rs | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/how-to.md b/docs/how-to.md index cac141c0..5f906a84 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -153,7 +153,7 @@ You can also install EQL by running [the installation script](https://github.com Once you have installed EQL, you can see what version is installed by querying the database: ```sql -SELECT cs_eql_version(); +SELECT eql_v1.version(); ``` This will output the version of EQL installed. diff --git a/packages/cipherstash-proxy/src/encrypt/mod.rs b/packages/cipherstash-proxy/src/encrypt/mod.rs index faac044e..0b82ff16 100644 --- a/packages/cipherstash-proxy/src/encrypt/mod.rs +++ b/packages/cipherstash-proxy/src/encrypt/mod.rs @@ -57,10 +57,11 @@ impl Encrypt { let eql_version = { let client = connect::database(&config.database).await?; - let rows = client.query("SELECT cs_eql_version();", &[]).await; + let rows = client.query("SELECT eql_v1.version() AS version;", &[]).await; + // let rows = client.query("SELECT 'WAT' AS version;", &[]).await; match rows { - Ok(rows) => rows.first().map(|row| row.get("cs_eql_version")), + Ok(rows) => rows.first().map(|row| row.get("version")), Err(err) => { warn!( msg = "Could not query EQL version from database",