diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 7fd37064..ac24e7de 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -34,6 +34,7 @@ jobs: CS_DEFAULT_KEYSET_ID: ${{ secrets.CS_DEFAULT_KEYSET_ID }} CS_CLIENT_ID: ${{ secrets.CS_CLIENT_ID }} CS_CLIENT_KEY: ${{ secrets.CS_CLIENT_KEY }} + CS_REGION: "ap-southeast-2.aws" RUST_BACKTRACE: "1" run: mise run benchmark:continuous # Download previous benchmark result from cache (if exists) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 64c17a0c..e1f8b028 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,11 +21,14 @@ jobs: - run: | mise run postgres:up --extra-args "--detach --wait" - env: + # REMEMBER TO ADD ENVIRONMENT VARIABLES TO tests/docker-compose.yml + # The tests/docker-compose.yml config passes the ENV vars into the container CS_WORKSPACE_ID: ${{ secrets.CS_WORKSPACE_ID }} CS_CLIENT_ACCESS_KEY: ${{ secrets.CS_CLIENT_ACCESS_KEY }} CS_DEFAULT_KEYSET_ID: ${{ secrets.CS_DEFAULT_KEYSET_ID }} CS_CLIENT_ID: ${{ secrets.CS_CLIENT_ID }} CS_CLIENT_KEY: ${{ secrets.CS_CLIENT_KEY }} + CS_REGION: "ap-southeast-2.aws" RUST_BACKTRACE: "1" run: | diff --git a/.gitignore b/.gitignore index a5b17a57..05413b30 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,6 @@ /cipherstash-proxy.local.toml mise.local.toml tests/pg/data** -tests/sql/cipherstash-encrypt.sql -tests/sql/cipherstash-encrypt-uninstall.sql .vscode rust-toolchain.toml @@ -13,8 +11,9 @@ rust-toolchain.toml # release artifacts /cipherstash-proxy -/cipherstash-eql.sql /packages/cipherstash-proxy/eql-version-at-build-time.txt +/cipherstash-encrypt.sql +/cipherstash-encrypt-uninstall.sql # credentials for local dev .env.proxy.docker diff --git a/Cargo.lock b/Cargo.lock index 70247c27..de9237ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -181,6 +181,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" dependencies = [ + "serde", "zeroize", ] @@ -193,7 +194,7 @@ dependencies = [ "asn1-rs-derive", "asn1-rs-impl", "displaydoc", - "nom", + "nom 7.1.3", "num-traits", "rusticata-macros", "thiserror 2.0.12", @@ -287,6 +288,61 @@ dependencies = [ "fs_extra", ] +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -317,6 +373,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +[[package]] +name = "base32" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "022dfe9eb35f19ebbcb51e0b40a5ab759f46ad60cadf7297e0bd085afb50e076" + [[package]] name = "base64" version = "0.22.1" @@ -537,7 +599,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -588,9 +650,9 @@ dependencies = [ [[package]] name = "cipherstash-client" -version = "0.18.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f099b1db6cf37b0ca36e9c8e0c2dade20f2035804e225f52475d44e750dd5dd5" +checksum = "0f85784b109d3cacec64a735ca5ac791ef4e9c4d1d451156dd3bb513c9b4ddf0" dependencies = [ "aes-gcm-siv", "anyhow", @@ -605,6 +667,7 @@ dependencies = [ "cipherstash-config", "cipherstash-core", "cllw-ore", + "cts-common", "derive_more", "dirs", "futures", @@ -679,7 +742,6 @@ dependencies = [ "bytes", "chrono", "cipherstash-client", - "cipherstash-config", "clap", "config", "eql-mapper", @@ -724,9 +786,12 @@ name = "cipherstash-proxy-integration" version = "0.1.0" dependencies = [ "chrono", + "cipherstash-client", + "cipherstash-config", "cipherstash-proxy", "clap", "fake 4.2.0", + "hex", "rand 0.9.0", "recipher", "rustls", @@ -739,6 +804,7 @@ dependencies = [ "tokio-rustls", "tracing", "tracing-subscriber", + "uuid", "webpki-roots", ] @@ -953,6 +1019,28 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "cts-common" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058540fce9a147af37cab4f55a5f9d8ae7f35f66efeec58c08cce6beb173d9c3" +dependencies = [ + "arrayvec", + "axum", + "base32", + "diesel", + "fake 3.1.0", + "http", + "miette", + "nom 8.0.0", + "rand 0.8.5", + "regex", + "serde", + "thiserror 1.0.69", + "url", + "vitaminc", +] + [[package]] name = "darling" version = "0.20.10" @@ -973,6 +1061,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", + "strsim", "syn 2.0.100", ] @@ -1022,7 +1111,7 @@ checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" dependencies = [ "asn1-rs", "displaydoc", - "nom", + "nom 7.1.3", "num-bigint", "num-traits", "rusticata-macros", @@ -1075,6 +1164,39 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc55fe0d1f6c107595572ec8b107c0999bb1a2e0b75e37429a4fb0d6474a0e7d" +[[package]] +name = "diesel" +version = "2.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff3e1edb1f37b4953dd5176916347289ed43d7119cc2e6c7c3f7849ff44ea506" +dependencies = [ + "chrono", + "diesel_derives", + "uuid", +] + +[[package]] +name = "diesel_derives" +version = "2.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68d4216021b3ea446fd2047f5c8f8fe6e98af34508a254a01e4d6bc1e844f84d" +dependencies = [ + "diesel_table_macro_syntax", + "dsl_auto_type", + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "diesel_table_macro_syntax" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "209c735641a413bc68c4923a9d6ad4bcb3ca306b794edaa7eb0b3228a99ffb25" +dependencies = [ + "syn 2.0.100", +] + [[package]] name = "diff" version = "0.1.13" @@ -1123,6 +1245,20 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "dsl_auto_type" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139ae9aca7527f85f26dd76483eb38533fd84bd571065da1739656ef71c5ff5b" +dependencies = [ + "darling", + "either", + "heck", + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "dummy" version = "0.8.0" @@ -1135,6 +1271,18 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "dummy" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abcba80bdf851db5616e27ff869399468e2d339d7c6480f5887681e6bdfc2186" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "dummy" version = "0.11.0" @@ -1172,6 +1320,7 @@ dependencies = [ "thiserror 2.0.12", "tracing", "tracing-subscriber", + "vec1", ] [[package]] @@ -1223,12 +1372,24 @@ dependencies = [ "uuid", ] +[[package]] +name = "fake" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef603df4ba9adbca6a332db7da6f614f21eafefbaf8e087844e452fdec152d0" +dependencies = [ + "deunicode", + "dummy 0.9.2", + "rand 0.8.5", +] + [[package]] name = "fake" version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b591050272097cc85b2f3c1cc4817ba4560057d10fcae6f7339f1cf622da0a0f" dependencies = [ + "chrono", "deunicode", "dummy 0.11.0", "rand 0.9.0", @@ -2034,6 +2195,12 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "md-5" version = "0.10.6" @@ -2187,6 +2354,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -3089,7 +3265,7 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -3328,6 +3504,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a" +dependencies = [ + "itoa", + "serde", +] + [[package]] name = "serde_spanned" version = "0.6.8" @@ -3905,6 +4091,7 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -4134,6 +4321,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "vec1" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eab68b56840f69efb0fefbe3ab6661499217ffdc58e2eef7c3f6f69835386322" + [[package]] name = "version_check" version = "0.9.5" @@ -4325,9 +4518,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.8" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9" +checksum = "2853738d1cc4f2da3a225c18ec6c3721abb31961096e9dbf5ab35fa88b19cfdb" dependencies = [ "rustls-pki-types", ] @@ -4727,7 +4920,7 @@ dependencies = [ "data-encoding", "der-parser", "lazy_static", - "nom", + "nom 7.1.3", "oid-registry", "rusticata-macros", "thiserror 2.0.12", diff --git a/docs/errors.md b/docs/errors.md index 7e5faceb..3ea90ec6 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -21,6 +21,10 @@ - [Unknown column](#encrypt-unknown-column) - [Unknown table](#encrypt-unknown-table) - [Unknown index term](#encrypt-unknown-index-term) + - [Column configuration mismatch](#encrypt-column-config-mismatch) + +- Decrypt errors: + - [Column could not be deserialised](#encrypt-column-could-not-be-deserialised) - Configuration errors: - [Missing or invalid TLS configuration](#config-missing-or-invalid-tls) @@ -263,9 +267,9 @@ The most likely cause is network access to the ZeroKMS service. ### How to Fix 1. Check that CipherStash ZeroKMS is available at [the status page](https://status.cipherstash.com/). -1. Check that CipherStash Proxy has network access to ZeroKMS in the appropriate region. +2. Check that CipherStash Proxy has network access to ZeroKMS in the appropriate region. -1. Check that the encrypted configuration `cast` matches the expected type. +3. Check that the encrypted configuration `cast` matches the expected type. @@ -307,14 +311,14 @@ For example: ### How to fix 1. Check the encrypted configuration has the correct type. -1. Check that the configuration has not changed. -1. Check [EQL](https://github.com/cipherstash/encrypt-query-language). +2. Check that the configuration has not changed. +3. Check [EQL](https://github.com/cipherstash/encrypt-query-language). ## Unknown Column -The column has an encrypted type (PostgreSQL `cs_encrypted_v1` type ) with no encryption configuration. +The column has an encrypted type (PostgreSQL `eql_v2_encrypted` type ) with no encryption configuration. Without the configuration, Cipherstash Proxy does not know how to encrypt the column. Any data is unprotected and unencrypted. @@ -331,7 +335,7 @@ Column 'column_name' in table 'table_name' has no Encrypt configuration 1. Define the encrypted configuration using [EQL](https://github.com/cipherstash/encrypt-query-language). -1. Add `users.email` as an encrypted column: +2. Add `users.email` as an encrypted column: ```sql SELECT cs_add_column_v1('users', 'email'); ``` @@ -341,7 +345,7 @@ Column 'column_name' in table 'table_name' has no Encrypt configuration ## Unknown Table -The table has one or more encrypted columns (PostgreSQL `cs_encrypted_v1` type ) with no encryption configuration. +The table has one or more encrypted columns (PostgreSQL `eql_v2_encrypted` type ) with no encryption configuration. Without the configuration, Cipherstash Proxy does not know how to encrypt the column. Any data is unprotected and unencrypted. @@ -357,7 +361,7 @@ Table 'table_name' has no Encrypt configuration 1. Define the encrypted configuration using [EQL](https://github.com/cipherstash/encrypt-query-language). -1. Add `users.email` as an encrypted column: +2. Add `users.email` as an encrypted column: ```sql SELECT cs_add_column_v1('users', 'email'); ``` @@ -385,13 +389,83 @@ Unknown Index Term for column '{column_name}' in table '{table_name}'. ### How to fix 1. Check the Encrypt configuration for the column. -1. Define the encrypted configuration using [EQL](https://github.com/cipherstash/encrypt-query-language). +2. Define the encrypted configuration using [EQL](https://github.com/cipherstash/encrypt-query-language). + + + + + + +## Column configuration mismatch + +A returned encrypted column does not match the column configuration. + +### Error message + +``` +Column configuration for column '{column_name}' in table '{table_name}' does not match the encrypted column. +``` + +### Notes + +CipherStash Proxy validates that encrypted columns match the configuration before decrypting any data. +If the table and column are not the same, this error is returned. +The check is there to help prevent "confused deputy" issues and the error should *never* appear during normal operation. + +If the error persists, please contact CipherStash [support](https://cipherstash.com/support). + + +### Further reading + +[AWS: The confused deputy problem](https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html) +[Wikipedia: Confused deputy problem](https://en.wikipedia.org/wiki/Confused_deputy_problem) + + + + + + + + + +# Decrypt errors + + +## Column could not be deserialised + +The column could not be deserialised for decryption. + + +### Error message + +``` +Column 'column_name' in table 'table_name' could not be deserialised. +``` + +### Notes + +CipherStash Proxy stores encrypted data and search terms as `jsonb`. The structure is defined as part of EQL. + +The error indicates an internal issue has occurred deserialising and extracting the ciphertext data for decryption. +It may be caused if the the encrypted data has been altered by another process or application. + +If the error persists, please contact CipherStash [support](https://cipherstash.com/support). + + +### How to Fix + +1. Check that the data in the encrypted column is in correct format [EQL](https://github.com/cipherstash/encrypt-query-language). + + + + + # Configuration errors diff --git a/docs/getting-started/schema-example.sql b/docs/getting-started/schema-example.sql index 29e3e743..e1627419 100644 --- a/docs/getting-started/schema-example.sql +++ b/docs/getting-started/schema-example.sql @@ -1,12 +1,12 @@ -TRUNCATE TABLE cs_configuration_v1; +TRUNCATE TABLE public.eql_v2_configuration; -- Exciting cipherstash table DROP TABLE IF EXISTS users; CREATE TABLE users ( id SERIAL PRIMARY KEY, - encrypted_email cs_encrypted_v1, - encrypted_dob cs_encrypted_v1, - encrypted_salary cs_encrypted_v1 + encrypted_email eql_v2_encrypted, + encrypted_dob eql_v2_encrypted, + encrypted_salary eql_v2_encrypted ); SELECT cs_add_index_v1( diff --git a/docs/how-to.md b/docs/how-to.md index a38f80cb..0f8954e5 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -153,7 +153,7 @@ You can also install EQL by running [the installation script](https://github.com Once you have installed EQL, you can see what version is installed by querying the database: ```sql -SELECT cs_eql_version(); +SELECT eql_v2.version(); ``` This will output the version of EQL installed. @@ -162,22 +162,22 @@ This will output the version of EQL installed. In your existing PostgreSQL database, you store your data in tables and columns. Those columns have types like `integer`, `text`, `timestamp`, and `boolean`. -When storing encrypted data in PostgreSQL with Proxy, you use a special column type called `cs_encrypted_v1`, which is [provided by EQL](#setting-up-the-database-schema). -`cs_encrypted_v1` is a container column type that can be used for any type of encrypted data you want to store or search, whether they are numbers (`int`, `small_int`, `big_int`), text (`text`), dates and times (`date`), or booleans (`boolean`). +When storing encrypted data in PostgreSQL with Proxy, you use a special column type called `eql_v2_encrypted`, which is [provided by EQL](#setting-up-the-database-schema). +`eql_v2_encrypted` is a container column type that can be used for any type of encrypted data you want to store or search, whether they are numbers (`int`, `small_int`, `big_int`), text (`text`), dates and times (`date`. `timestamp`), or booleans (`boolean`). Create a table with an encrypted column for `email`: ```sql CREATE TABLE users ( id SERIAL PRIMARY KEY, - email cs_encrypted_v1 + email eql_v2_encrypted ) ``` This creates a `users` table with two columns: - `id`, an autoincrementing integer column that is the primary key for the record - - `email`, a `cs_encrypted_v1` column + - `email`, a `eql_v2_encrypted` column There are important differences between the plaintext columns you've traditionally used in PostgreSQL and encrypted columns with CipherStash Proxy: diff --git a/mise.toml b/mise.toml index 9cf04e2a..cd840193 100644 --- a/mise.toml +++ b/mise.toml @@ -22,8 +22,9 @@ CS_DATABASE__PORT = "5532" # Default configuration for dev cipherstash-proxy run using 'mise run proxy:up' CS_PROXY__HOST = "proxy" # Misc -DOCKER_CLI_HINTS = "false" # Please don't show us What's Next. -CS_EQL_VERSION = "eql-1.0.1" +DOCKER_CLI_HINTS = "false" # Please don't show us What's Next. + +CS_EQL_VERSION = "eql-2.0.0" [tools] "cargo:cargo-binstall" = "latest" @@ -409,27 +410,28 @@ fi """ [tasks."postgres:setup"] +depends = ["postgres:eql:teardown"] alias = 's' description = "Installs EQL and applies schema to database" run = """ #!/bin/bash cd tests mise run postgres:fail_if_not_running -mise run postgres:eql:download -cat sql/cipherstash-encrypt.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- +cat sql/schema-uninstall.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- +cat ../cipherstash-encrypt-uninstall.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- +cat ../cipherstash-encrypt.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- cat sql/schema.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- """ [tasks."postgres:eql:teardown"] -alias = 's' +depends = ["eql:download"] description = "Uninstalls EQL and removes schema from database" run = """ #!/bin/bash cd tests mise run postgres:fail_if_not_running -mise run postgres:eql:download cat sql/schema-uninstall.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- -cat sql/cipherstash-encrypt-uninstall.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- +cat ../cipherstash-encrypt-uninstall.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f- """ [tasks."postgres:up"] @@ -490,34 +492,34 @@ for d in tests/pg/data-*; do done """ - -[tasks."postgres:eql:download"] +[tasks."eql:download"] alias = 'e' -description = "Download latest EQL release" +description = "Download latest EQL release or use local copy" dir = "{{config_root}}/tests" outputs = [ - "{{config_root}}/tests/sql/cipherstash-encrypt.sql", - "{{config_root}}/tests/sql/cipherstash-encrypt-uninstall.sql", + "{{config_root}}/cipherstash-encrypt.sql", + "{{config_root}}/cipherstash-encrypt-uninstall.sql", ] run = """ # install script if [ -z "$CS_EQL_PATH" ]; then - curl -sLo sql/cipherstash-encrypt.sql https://github.com/cipherstash/encrypt-query-language/releases/download/${CS_EQL_VERSION}/cipherstash-encrypt.sql + echo "Downloading ${CS_EQL_VERSION} install" + curl -sLo "{{config_root}}/cipherstash-encrypt.sql" https://github.com/cipherstash/encrypt-query-language/releases/download/${CS_EQL_VERSION}/cipherstash-encrypt.sql else - echo "Using EQL: ${CS_EQL_PATH}" - cp "$CS_EQL_PATH" sql/cipherstash-encrypt.sql + echo "Using EQL: ${CS_EQL_PATH}/cipherstash-encrypt.sql" + cp "$CS_EQL_PATH/cipherstash-encrypt.sql" "{{config_root}}/cipherstash-encrypt.sql" fi # uninstall script -if [ -z "$CS_EQL_UNINSTALL_PATH" ]; then - curl -sLo sql/cipherstash-encrypt-uninstall.sql https://github.com/cipherstash/encrypt-query-language/releases/download/${CS_EQL_VERSION}/cipherstash-encrypt-uninstall.sql +if [ -z "$CS_EQL_PATH" ]; then + echo "Downloading ${CS_EQL_VERSION} uninstall" + curl -sLo "{{config_root}}/cipherstash-encrypt-uninstall.sql" https://github.com/cipherstash/encrypt-query-language/releases/download/${CS_EQL_VERSION}/cipherstash-encrypt-uninstall.sql else - echo "Using EQL: ${CS_EQL_PATH}" - cp "$CS_EQL_UNINSTALL_PATH" sql/cipherstash-encrypt-uninstall.sql + echo "Using EQL: ${CS_EQL_PATH}/cipherstash-encrypt-uninstall.sql" + cp "$CS_EQL_PATH/cipherstash-encrypt-uninstall.sql" "{{config_root}}/cipherstash-encrypt-uninstall.sql" fi """ - [tasks."python:test"] dir = "{{config_root}}/tests" description = "Runs python tests" @@ -567,7 +569,7 @@ cp -v {{config_root}}/target/{{ target }}/release/cipherstash-proxy {{config_roo """ [tasks."build:docker"] -depends = ["build:docker:fetch_eql"] +depends = ["eql:download"] description = "Build a Docker image for cipherstash-proxy" run = """ {% set default_platform = "linux/" ~ arch() | replace(from="x86_64", to="amd64") %} diff --git a/packages/cipherstash-proxy-integration/Cargo.toml b/packages/cipherstash-proxy-integration/Cargo.toml index 546e1ff8..90162eb9 100644 --- a/packages/cipherstash-proxy-integration/Cargo.toml +++ b/packages/cipherstash-proxy-integration/Cargo.toml @@ -21,8 +21,12 @@ tokio-postgres-rustls = "0.13.0" tokio-rustls = "0.26.0" tracing = { workspace = true } tracing-subscriber = { workspace = true } -webpki-roots = "0.26.7" +webpki-roots = "1.0" [dev-dependencies] +cipherstash-client = { version = "0.22.0", features = ["tokio"] } +cipherstash-config = "0.2.3" clap = "4.5.32" -fake = { version = "4", features = ["derive"] } +fake = { version = "4", features = ["chrono", "derive"] } +hex = "0.4.3" +uuid = { version = "1.11.0", features = ["serde", "v4"] } diff --git a/packages/cipherstash-proxy-integration/src/extended_protocol_error_messages.rs b/packages/cipherstash-proxy-integration/src/extended_protocol_error_messages.rs index c45ec4af..ae868aea 100644 --- a/packages/cipherstash-proxy-integration/src/extended_protocol_error_messages.rs +++ b/packages/cipherstash-proxy-integration/src/extended_protocol_error_messages.rs @@ -66,11 +66,7 @@ mod tests { if let Err(err) = result { let msg = err.to_string(); - // This is similar to below. The error message comes from tokio-postgres when Proxy - // returns cs_encrypted_v1 and the client cannot convert to a string. - // If mapping errors are enabled (enable_mapping_errors or CS_DEVELOPMENT__ENABLE_MAPPING_ERRORS), - // then Proxy will return an error that says "Column X in table Y has no Encrypt configuration" - assert_eq!(msg, "error serializing parameter 1: cannot convert between the Rust type `&str` and the Postgres type `cs_encrypted_v1`"); + assert_eq!(msg, "db error: ERROR: Column 'encrypted_unconfigured' in table 'unconfigured' has no Encrypt configuration. For help visit https://github.com/cipherstash/proxy/blob/main/docs/errors.md#encrypt-unknown-column"); } else { unreachable!(); } diff --git a/packages/cipherstash-proxy-integration/src/generate.rs b/packages/cipherstash-proxy-integration/src/generate.rs new file mode 100644 index 00000000..13d11820 --- /dev/null +++ b/packages/cipherstash-proxy-integration/src/generate.rs @@ -0,0 +1,259 @@ +#[cfg(test)] +mod tests { + use crate::common::trace; + use cipherstash_client::config::EnvSource; + use cipherstash_client::credentials::auto_refresh::AutoRefresh; + use cipherstash_client::encryption::{ + Encrypted, EncryptedSteVecTerm, JsonIndexer, JsonIndexerOptions, OreTerm, Plaintext, + PlaintextTarget, ReferencedPendingPipeline, + }; + use cipherstash_client::{encryption::ScopedCipher, zerokms::EncryptedRecord}; + use cipherstash_client::{ConsoleConfig, CtsConfig, ZeroKMSConfig}; + use cipherstash_config::column::{Index, IndexType}; + use cipherstash_config::{ColumnConfig, ColumnType}; + use cipherstash_proxy::Identifier; + use serde::{Deserialize, Serialize}; + use std::sync::Arc; + use tracing::info; + use uuid::Uuid; + + pub mod option_mp_base85 { + use cipherstash_client::zerokms::encrypted_record::formats::mp_base85; + use cipherstash_client::zerokms::EncryptedRecord; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize( + value: &Option, + serializer: S, + ) -> Result + where + S: Serializer, + { + match value { + Some(record) => mp_base85::serialize(record, serializer), + None => serializer.serialize_none(), + } + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let result = Option::::deserialize(deserializer)?; + Ok(result) + } + } + + #[derive(Debug, Deserialize, Serialize)] + pub struct EqlEncrypted { + #[serde(rename = "c", with = "option_mp_base85")] + ciphertext: Option, + #[serde(rename = "i")] + identifier: Identifier, + #[serde(rename = "v")] + version: u16, + + #[serde(rename = "o")] + ore_index: Option>, + #[serde(rename = "m")] + match_index: Option>, + #[serde(rename = "u")] + unique_index: Option, + + #[serde(rename = "s")] + selector: Option, + + #[serde(rename = "b")] + blake3_index: Option, + + #[serde(rename = "ocf")] + ore_cclw_fixed_index: Option, + #[serde(rename = "ocv")] + ore_cclw_var_index: Option, + + #[serde(rename = "sv")] + ste_vec_index: Option>, + } + + #[derive(Debug, Deserialize, Serialize)] + pub struct EqlSteVecEncrypted { + #[serde(rename = "c", with = "option_mp_base85")] + ciphertext: Option, + + #[serde(rename = "s")] + selector: Option, + #[serde(rename = "b")] + blake3_index: Option, + #[serde(rename = "ocf")] + ore_cclw_fixed_index: Option, + #[serde(rename = "ocv")] + ore_cclw_var_index: Option, + } + + impl EqlEncrypted { + pub fn ste_vec(ste_vec_index: Vec) -> Self { + Self { + ste_vec_index: Some(ste_vec_index), + ciphertext: None, + identifier: Identifier { + table: "blah".to_string(), + column: "vtha".to_string(), + }, + version: 1, + ore_index: None, + match_index: None, + unique_index: None, + selector: None, + ore_cclw_fixed_index: None, + ore_cclw_var_index: None, + blake3_index: None, + } + } + } + impl EqlSteVecEncrypted { + pub fn ste_vec_element(selector: String, record: EncryptedRecord) -> Self { + Self { + ciphertext: Some(record), + selector: Some(selector), + ore_cclw_fixed_index: None, + ore_cclw_var_index: None, + blake3_index: None, + } + } + } + + #[tokio::test] + async fn generate_ste_vec() { + trace(); + + // clear().await; + // let client = connect_with_tls(PROXY).await; + + let console_config = ConsoleConfig::builder().with_env().build().unwrap(); + let cts_config = CtsConfig::builder().with_env().build().unwrap(); + let zerokms_config = ZeroKMSConfig::builder() + .add_source(EnvSource::default()) + .console_config(&console_config) + .cts_config(&cts_config) + .build_with_client_key() + .unwrap(); + let zerokms_client = zerokms_config + .create_client_with_credentials(AutoRefresh::new(zerokms_config.credentials())); + + let dataset_id = Uuid::parse_str("295504329cb045c398dc464c52a287a1").unwrap(); + + let cipher = Arc::new( + ScopedCipher::init(Arc::new(zerokms_client), Some(dataset_id)) + .await + .unwrap(), + ); + + let prefix = "prefix".to_string(); + + let column_config = ColumnConfig::build("column_name".to_string()) + .casts_as(ColumnType::JsonB) + .add_index(Index::new(IndexType::SteVec { + prefix: prefix.to_owned(), + })); + + // let mut value = + // serde_json::from_str::("{\"hello\": \"one\", \"n\": 10}").unwrap(); + + // let mut value = + // serde_json::from_str::("{\"hello\": \"two\", \"n\": 20}").unwrap(); + + let value = + serde_json::from_str::("{\"hello\": \"two\", \"n\": 30}").unwrap(); + + // let mut value = + // serde_json::from_str::("{\"hello\": \"world\", \"n\": 42}").unwrap(); + + // let mut value = + // serde_json::from_str::("{\"hello\": \"world\", \"n\": 42}").unwrap(); + + // let mut value = + // serde_json::from_str::("{\"blah\": { \"vtha\": 42 }}").unwrap(); + + let plaintext = Plaintext::JsonB(Some(value)); + + let idx = 0; + + let mut pipeline = ReferencedPendingPipeline::new(cipher.clone()); + let encryptable = PlaintextTarget::new(plaintext, column_config); + pipeline + .add_with_ref::(encryptable, idx) + .unwrap(); + + let mut encrypteds = vec![]; + + let mut result = pipeline.encrypt(None).await.unwrap(); + if let Some(Encrypted::SteVec(ste_vec)) = result.remove(idx) { + for entry in ste_vec { + let selector = hex::encode(entry.0.as_bytes()); + let term = entry.1; + let record = entry.2; + + let mut e = EqlSteVecEncrypted::ste_vec_element(selector, record); + + match term { + EncryptedSteVecTerm::Mac(items) => { + e.blake3_index = Some(hex::encode(&items)); + } + EncryptedSteVecTerm::OreFixed(o) => { + e.ore_cclw_fixed_index = Some(hex::encode(&o)); + } + EncryptedSteVecTerm::OreVariable(o) => { + e.ore_cclw_var_index = Some(hex::encode(&o)); + } + } + + encrypteds.push(e); + } + // info!("{:?}" = encrypteds); + } + + info!("---------------------------------------------"); + + let e = EqlEncrypted::ste_vec(encrypteds); + info!("{:?}" = ?e); + + let json = serde_json::to_value(e).unwrap(); + info!("{}", json); + + let indexer = JsonIndexer::new(JsonIndexerOptions { prefix }); + + info!("---------------------------------------------"); + + // Path + // let path: String = "$.blah.vtha".to_string(); + // let selector = Selector::parse(&path).unwrap(); + // let selector = indexer.generate_selector(selector, cipher.index_key()); + // let selector = hex::encode(selector.0); + // info!("{}", selector); + + // Comparison + let n = 30; + let term = OreTerm::Number(n); + + let term = indexer.generate_term(term, cipher.index_key()).unwrap(); + + match term { + EncryptedSteVecTerm::Mac(_) => todo!(), + EncryptedSteVecTerm::OreFixed(ore_cllw8_v1) => { + let term = hex::encode(ore_cllw8_v1.bytes); + info!("{n}: {term}"); + } + EncryptedSteVecTerm::OreVariable(_) => todo!(), + } + + // if let Some(ste_vec_index) = e.ste_vec_index { + // for e in ste_vec_index { + // info!("{}", e); + // if let Some(ct) = e.ciphertext { + // let decrypted = cipher.decrypt(encrypted).await?; + // info!("{}", decrypted); + // } + // } + // } + } +} diff --git a/packages/cipherstash-proxy-integration/src/map_concat.rs b/packages/cipherstash-proxy-integration/src/map_concat.rs index 04005227..90abd9e4 100644 --- a/packages/cipherstash-proxy-integration/src/map_concat.rs +++ b/packages/cipherstash-proxy-integration/src/map_concat.rs @@ -1,11 +1,19 @@ #[cfg(test)] mod tests { - use crate::common::{connect_with_tls, PROXY}; + use crate::common::{clear, connect_with_tls, id, PROXY}; #[tokio::test] async fn map_concat_regression() { let client = connect_with_tls(PROXY).await; + clear().await; + + let id = id(); + let encrypted_text = "hello@cipherstash.com"; + + let sql = "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)"; + client.query(sql, &[&id, &encrypted_text]).await.unwrap(); + let sql = "UPDATE encrypted SET encrypted_text = encrypted_text || 'suffix';"; client diff --git a/packages/cipherstash-proxy/Cargo.toml b/packages/cipherstash-proxy/Cargo.toml index d63a2513..5808db9e 100644 --- a/packages/cipherstash-proxy/Cargo.toml +++ b/packages/cipherstash-proxy/Cargo.toml @@ -8,8 +8,7 @@ bigdecimal = { version = "0.4.6", features = ["serde-json"] } arc-swap = "1.7.1" bytes = { version = "1.9", default-features = false } chrono = { version = "0.4.39", features = ["clock"] } -cipherstash-client = { version = "0.18.0-pre.1", features = ["tokio"] } -cipherstash-config = "0.2.3" +cipherstash-client = { version = "0.22.0", features = ["tokio"] } clap = { version = "4.5.31", features = ["derive", "env"] } config = { version = "0.15", features = [ "async", diff --git a/packages/cipherstash-proxy/src/config/log.rs b/packages/cipherstash-proxy/src/config/log.rs index 06c235ce..aa5ebbc4 100644 --- a/packages/cipherstash-proxy/src/config/log.rs +++ b/packages/cipherstash-proxy/src/config/log.rs @@ -35,6 +35,9 @@ pub struct LogConfig { #[serde(default = "LogConfig::default_log_level")] pub encrypt_level: LogLevel, + #[serde(default = "LogConfig::default_log_level")] + pub decrypt_level: LogLevel, + #[serde(default = "LogConfig::default_log_level")] pub encrypt_config_level: LogLevel, @@ -107,24 +110,7 @@ impl Display for LogLevel { impl Default for LogConfig { fn default() -> Self { - LogConfig { - format: LogConfig::default_log_format(), - output: LogConfig::default_log_output(), - ansi_enabled: LogConfig::default_ansi_enabled(), - level: LogConfig::default_log_level(), - development_level: LogConfig::default_log_level(), - authentication_level: LogConfig::default_log_level(), - context_level: LogConfig::default_log_level(), - encrypt_level: LogConfig::default_log_level(), - encoding_level: LogConfig::default_log_level(), - encrypt_config_level: LogConfig::default_log_level(), - keyset_level: LogConfig::default_log_level(), - migrate_level: LogConfig::default_log_level(), - protocol_level: LogConfig::default_log_level(), - mapper_level: LogConfig::default_log_level(), - schema_level: LogConfig::default_log_level(), - config_level: LogConfig::default_log_level(), - } + Self::with_level(LogConfig::default_log_level()) } } @@ -141,6 +127,7 @@ impl LogConfig { encoding_level: level, encrypt_level: level, encrypt_config_level: level, + decrypt_level: level, keyset_level: level, migrate_level: level, protocol_level: level, diff --git a/packages/cipherstash-proxy/src/connect/async_stream.rs b/packages/cipherstash-proxy/src/connect/async_stream.rs index 2c0b6b2e..9daa1058 100644 --- a/packages/cipherstash-proxy/src/connect/async_stream.rs +++ b/packages/cipherstash-proxy/src/connect/async_stream.rs @@ -23,7 +23,7 @@ use x509_parser::prelude::{FromDer, X509Certificate}; #[derive(Debug)] pub enum AsyncStream { Tcp(TcpStream), - Tls(TlsStream), + Tls(Box>), } impl AsyncStream { diff --git a/packages/cipherstash-proxy/src/encrypt/config/encrypt_config.rs b/packages/cipherstash-proxy/src/encrypt/config/encrypt_config.rs index c9bf5101..730e678e 100644 --- a/packages/cipherstash-proxy/src/encrypt/config/encrypt_config.rs +++ b/packages/cipherstash-proxy/src/encrypt/config/encrypt_config.rs @@ -3,7 +3,7 @@ use crate::{ error::{ConfigError, Error}, log::KEYSET, }; -use cipherstash_config::{ +use cipherstash_client::schema::{ column::{Index, IndexType, TokenFilter, Tokenizer}, ColumnConfig, ColumnType, }; diff --git a/packages/cipherstash-proxy/src/encrypt/config/manager.rs b/packages/cipherstash-proxy/src/encrypt/config/manager.rs index 31312533..792f64eb 100644 --- a/packages/cipherstash-proxy/src/encrypt/config/manager.rs +++ b/packages/cipherstash-proxy/src/encrypt/config/manager.rs @@ -7,7 +7,7 @@ use crate::{ log::ENCRYPT_CONFIG, }; use arc_swap::ArcSwap; -use cipherstash_config::ColumnConfig; +use cipherstash_client::schema::ColumnConfig; use serde_json::Value; use std::{collections::HashMap, sync::Arc, time::Duration}; use tokio::{task::JoinHandle, time}; @@ -195,8 +195,7 @@ pub async fn load_encrypt_config(config: &DatabaseConfig) -> Result bool { let msg = e.to_string(); - msg.contains("cs_configuration_v1") && msg.contains("does not exist") + msg.contains("eql_v2_configuration") && msg.contains("does not exist") } diff --git a/packages/cipherstash-proxy/src/encrypt/mod.rs b/packages/cipherstash-proxy/src/encrypt/mod.rs index faac044e..826c3998 100644 --- a/packages/cipherstash-proxy/src/encrypt/mod.rs +++ b/packages/cipherstash-proxy/src/encrypt/mod.rs @@ -3,7 +3,8 @@ mod schema; use crate::{ config::TandemConfig, - connect, eql, + connect, + eql::{self, EqlEncryptedBody, EqlEncryptedIndexes}, error::{EncryptError, Error}, log::ENCRYPT, postgresql::Column, @@ -13,13 +14,12 @@ use cipherstash_client::{ config::EnvSource, credentials::{auto_refresh::AutoRefresh, ServiceCredentials}, encryption::{ - self, Encrypted, EncryptionError, IndexTerm, Plaintext, PlaintextTarget, - ReferencedPendingPipeline, + self, Encrypted, EncryptedEntry, EncryptedSteVecTerm, IndexTerm, Plaintext, + PlaintextTarget, ReferencedPendingPipeline, }, - zerokms::EncryptedRecord, + schema::ColumnConfig, ConsoleConfig, CtsConfig, ZeroKMSConfig, }; -use cipherstash_config::ColumnConfig; use config::EncryptConfigManager; use schema::SchemaManager; use std::{sync::Arc, vec}; @@ -57,10 +57,12 @@ impl Encrypt { let eql_version = { let client = connect::database(&config.database).await?; - let rows = client.query("SELECT cs_eql_version();", &[]).await; + let rows = client + .query("SELECT eql_v2.version() AS version;", &[]) + .await; match rows { - Ok(rows) => rows.first().map(|row| row.get("cs_eql_version")), + Ok(rows) => rows.first().map(|row| row.get("version")), Err(err) => { warn!( msg = "Could not query EQL version from database", @@ -88,7 +90,7 @@ impl Encrypt { &self, plaintexts: Vec>, columns: &[Option], - ) -> Result>, Error> { + ) -> Result>, Error> { let mut pipeline = ReferencedPendingPipeline::new(self.cipher.clone()); for (idx, item) in plaintexts.into_iter().zip(columns.iter()).enumerate() { @@ -141,22 +143,17 @@ impl Encrypt { /// pub async fn decrypt( &self, - ciphertexts: Vec>, + ciphertexts: Vec>, ) -> Result>, Error> { // Create a mutable vector to hold the decrypted results let mut results = vec![None; ciphertexts.len()]; // Collect the index and ciphertext details for every Some(ciphertext) - let (indices, encrypted) = ciphertexts + let (indices, encrypted): (Vec<_>, Vec<_>) = ciphertexts .into_iter() .enumerate() - .filter_map(|(idx, opt)| { - opt.map(|ct| { - eql_encrypted_to_encrypted_record(ct) - .map(|encrypted_record| (idx, encrypted_record)) - }) - }) - .collect::, Vec<_>), _>>()?; + .filter_map(|(idx, eql)| Some((idx, eql?.body.ciphertext))) + .collect::<_>(); // Decrypt the ciphertexts let decrypted = self.cipher.decrypt(encrypted).await?; @@ -203,7 +200,14 @@ async fn init_cipher(config: &TandemConfig) -> Result { // Not using with_env because the proxy config should take precedence let builder = ZeroKMSConfig::builder() .add_source(EnvSource::default()) - .workspace_id(&config.auth.workspace_id) + .workspace_id( + config + .auth + .workspace_id + .to_owned() + .try_into() + .map_err(cipherstash_client::config::ConfigError::from)?, + ) .access_key(&config.auth.client_access_key) .try_with_client_id(&config.encrypt.client_id)? .try_with_client_key(&config.encrypt.client_key)? @@ -236,56 +240,130 @@ async fn init_cipher(config: &TandemConfig) -> Result { fn to_eql_encrypted( encrypted: Encrypted, identifier: &Identifier, -) -> Result { +) -> Result { debug!(target: ENCRYPT, msg = "Encrypted to EQL", ?identifier); match encrypted { Encrypted::Record(ciphertext, terms) => { - struct Indexes { - match_index: Option>, - ore_index: Option>, - unique_index: Option, - } - - let mut indexes = Indexes { - match_index: None, - ore_index: None, - unique_index: None, - }; + let mut match_index: Option> = None; + let mut ore_index: Option> = None; + let mut unique_index: Option = None; + let mut blake3_index: Option = None; + let mut ore_cclw_fixed_index: Option = None; + let mut ore_cclw_var_index: Option = None; + let mut selector: Option = None; for index_term in terms { match index_term { IndexTerm::Binary(bytes) => { - indexes.unique_index = Some(format_index_term_binary(&bytes)) + unique_index = Some(format_index_term_binary(&bytes)) } - IndexTerm::BitMap(inner) => indexes.match_index = Some(inner), - IndexTerm::OreArray(vec_of_bytes) => { - indexes.ore_index = Some(format_index_term_ore_array(&vec_of_bytes)); + IndexTerm::BitMap(inner) => match_index = Some(inner), + IndexTerm::OreArray(bytes) => { + ore_index = Some(format_index_term_ore_array(&bytes)); } IndexTerm::OreFull(bytes) => { - indexes.ore_index = Some(format_index_term_ore(&bytes)); + ore_index = Some(format_index_term_ore(&bytes)); } IndexTerm::OreLeft(bytes) => { - indexes.ore_index = Some(format_index_term_ore(&bytes)); + ore_index = Some(format_index_term_ore(&bytes)); } + IndexTerm::BinaryVec(_) => todo!(), + IndexTerm::SteVecSelector(s) => { + selector = Some(hex::encode(s.as_bytes())); + } + IndexTerm::SteVecTerm(ste_vec_term) => match ste_vec_term { + EncryptedSteVecTerm::Mac(bytes) => blake3_index = Some(hex::encode(bytes)), + EncryptedSteVecTerm::OreFixed(ore) => { + ore_cclw_fixed_index = Some(hex::encode(&ore)) + } + EncryptedSteVecTerm::OreVariable(ore) => { + ore_cclw_var_index = Some(hex::encode(&ore)) + } + }, + IndexTerm::SteQueryVec(_query) => {} // TODO: what do we do here? IndexTerm::Null => {} - _ => return Err(EncryptError::UnknownIndexTerm(identifier.to_owned()).into()), }; } - Ok(eql::Encrypted::Ciphertext { - ciphertext, + Ok(eql::EqlEncrypted { identifier: identifier.to_owned(), - match_index: indexes.match_index, - ore_index: indexes.ore_index, - unique_index: indexes.unique_index, version: 1, + body: EqlEncryptedBody { + ciphertext, + indexes: EqlEncryptedIndexes { + match_index, + ore_index, + unique_index, + blake3_index, + ore_cclw_fixed_index, + ore_cclw_var_index, + selector, + ste_vec_index: None, + }, + is_array_item: None, + }, + }) + } + Encrypted::SteVec(ste_vec) => { + let ciphertext = ste_vec.root_ciphertext()?.clone(); + + let ste_vec_index: Vec = ste_vec + .into_iter() + .map( + |EncryptedEntry { + tokenized_selector, + term, + record, + parent_is_array, + }| { + let indexes = match term { + EncryptedSteVecTerm::Mac(bytes) => EqlEncryptedIndexes { + selector: Some(hex::encode(tokenized_selector.as_bytes())), + blake3_index: Some(hex::encode(bytes)), + ..Default::default() + }, + EncryptedSteVecTerm::OreFixed(ore) => EqlEncryptedIndexes { + selector: Some(hex::encode(tokenized_selector.as_bytes())), + ore_cclw_fixed_index: Some(hex::encode(&ore)), + ..Default::default() + }, + EncryptedSteVecTerm::OreVariable(ore) => EqlEncryptedIndexes { + selector: Some(hex::encode(tokenized_selector.as_bytes())), + ore_cclw_var_index: Some(hex::encode(&ore)), + ..Default::default() + }, + }; + + eql::EqlEncryptedBody { + ciphertext: record, + indexes, + is_array_item: Some(parent_is_array), + } + }, + ) + .collect(); + + // FIXME: I'm unsure if I've handled the root ciphertext correctly + // The way it's implemented right now is that it will be repeated one in the ste_vec_index. + Ok(eql::EqlEncrypted { + identifier: identifier.to_owned(), + version: 1, + body: EqlEncryptedBody { + ciphertext: ciphertext.clone(), + indexes: EqlEncryptedIndexes { + match_index: None, + ore_index: None, + unique_index: None, + blake3_index: None, + ore_cclw_fixed_index: None, + ore_cclw_var_index: None, + selector: None, + ste_vec_index: Some(ste_vec_index), + }, + is_array_item: None, + }, }) } - Encrypted::SteVec(ste_vec_index) => Ok(eql::Encrypted::SteVec { - identifier: identifier.to_owned(), - ste_vec_index, - version: 1, - }), } } @@ -314,15 +392,6 @@ fn format_index_term_ore(bytes: &Vec) -> Vec { vec![format_index_term_ore_bytea(bytes)] } -fn eql_encrypted_to_encrypted_record( - eql_encrypted: eql::Encrypted, -) -> Result { - match eql_encrypted { - eql::Encrypted::Ciphertext { ciphertext, .. } => Ok(ciphertext), - eql::Encrypted::SteVec { ste_vec_index, .. } => ste_vec_index.into_root_ciphertext(), - } -} - fn plaintext_type_name(pt: Plaintext) -> String { match pt { Plaintext::BigInt(_) => "BigInt".to_string(), diff --git a/packages/cipherstash-proxy/src/encrypt/schema/manager.rs b/packages/cipherstash-proxy/src/encrypt/schema/manager.rs index fc000ccb..9a15f77f 100644 --- a/packages/cipherstash-proxy/src/encrypt/schema/manager.rs +++ b/packages/cipherstash-proxy/src/encrypt/schema/manager.rs @@ -132,19 +132,18 @@ pub async fn load_schema(config: &DatabaseConfig) -> Result { let table_name: String = table.get("table_name"); let primary_keys: Vec> = table.get("primary_keys"); let columns: Vec = table.get("columns"); - let _types: Vec> = table.get("column_types"); - let domains: Vec> = table.get("column_domains"); + let column_type_names: Vec> = table.get("column_type_names"); let mut table = Table::new(Ident::new(&table_name)); - columns.iter().zip(domains).for_each(|(col, domain)| { + columns.iter().zip(column_type_names).for_each(|(col, column_type_name)| { let is_primary_key = primary_keys.contains(&Some(col.to_string())); let ident = Ident::with_quote('"', col); - let column = match domain.as_deref() { - Some("cs_encrypted_v1") => { - debug!(target: SCHEMA, msg = "cs_encrypted_v1 column", table = table_name, column = col); + let column = match column_type_name.as_deref() { + Some("eql_v2_encrypted") => { + debug!(target: SCHEMA, msg = "eql_v2_encrypted column", table = table_name, column = col); Column::eql(ident) } _ => Column::native(ident), diff --git a/packages/cipherstash-proxy/src/encrypt/sql/select_config.sql b/packages/cipherstash-proxy/src/encrypt/sql/select_config.sql index 72827f37..b748a9d6 100644 --- a/packages/cipherstash-proxy/src/encrypt/sql/select_config.sql +++ b/packages/cipherstash-proxy/src/encrypt/sql/select_config.sql @@ -1 +1 @@ -SELECT data FROM cs_configuration_v1 WHERE state = 'active' LIMIT 1; +SELECT data FROM public.eql_v2_configuration WHERE state = 'active' LIMIT 1; diff --git a/packages/cipherstash-proxy/src/encrypt/sql/select_table_schemas.sql b/packages/cipherstash-proxy/src/encrypt/sql/select_table_schemas.sql index ee3ba513..88743f3e 100644 --- a/packages/cipherstash-proxy/src/encrypt/sql/select_table_schemas.sql +++ b/packages/cipherstash-proxy/src/encrypt/sql/select_table_schemas.sql @@ -3,8 +3,7 @@ SELECT t.table_name, array_agg(distinct k.column_name)::text[] AS primary_keys, array_agg(c.column_name)::text[] AS columns, - array_agg(c.data_type)::text[] AS column_types, - array_agg(c.domain_name)::text[] AS column_domains + array_agg(c.udt_name)::text[] AS column_type_names FROM information_schema.tables t LEFT JOIN @@ -24,3 +23,6 @@ GROUP BY t.table_schema, t.table_name ORDER BY t.table_schema, t.table_name; + + + diff --git a/packages/cipherstash-proxy/src/eql/mod.rs b/packages/cipherstash-proxy/src/eql/mod.rs index 69266310..e5ef20ea 100644 --- a/packages/cipherstash-proxy/src/eql/mod.rs +++ b/packages/cipherstash-proxy/src/eql/mod.rs @@ -1,7 +1,4 @@ -use cipherstash_client::{ - encryption::SteVec, - zerokms::{encrypted_record, EncryptedRecord}, -}; +use cipherstash_client::zerokms::{encrypted_record, EncryptedRecord}; use serde::{Deserialize, Serialize}; use sqltk::parser::ast::Ident; @@ -16,6 +13,7 @@ pub struct Plaintext { #[serde(rename = "q")] pub for_query: Option, } + #[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] pub struct Identifier { #[serde(rename = "t")] @@ -65,54 +63,60 @@ pub enum ForQuery { } #[derive(Debug, Deserialize, Serialize)] -#[serde(tag = "k")] -pub enum Encrypted { - #[serde(rename = "ct")] - Ciphertext { - #[serde(rename = "c", with = "encrypted_record::formats::mp_base85")] - ciphertext: EncryptedRecord, - #[serde(rename = "o")] - ore_index: Option>, - #[serde(rename = "m")] - match_index: Option>, - #[serde(rename = "u")] - unique_index: Option, - #[serde(rename = "i")] - identifier: Identifier, - #[serde(rename = "v")] - version: u16, - }, - #[serde(rename = "sv")] - SteVec { - #[serde(rename = "sv")] - ste_vec_index: SteVec<16>, - #[serde(rename = "i")] - identifier: Identifier, - #[serde(rename = "v")] - version: u16, - }, +pub struct EqlEncrypted { + #[serde(rename = "i")] + pub(crate) identifier: Identifier, + #[serde(rename = "v")] + pub(crate) version: u16, + + #[serde(flatten)] + pub(crate) body: EqlEncryptedBody, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct EqlEncryptedBody { + #[serde(rename = "c", with = "encrypted_record::formats::mp_base85")] + pub(crate) ciphertext: EncryptedRecord, + + #[serde(flatten)] + pub(crate) indexes: EqlEncryptedIndexes, + + #[serde(rename = "a", skip_serializing_if = "Option::is_none")] + pub(crate) is_array_item: Option, } -// fn ident_de<'de, D>(deserializer: D) -> Result -// where -// D: serde::Deserializer<'de>, -// { -// let s = String::deserialize(deserializer)?; -// Ok(Ident::with_quote('"', s)) -// } - -// fn ident_se(ident: &Ident, serializer: S) -> Result -// where -// S: Serializer, -// { -// let s = ident.to_string(); -// serializer.serialize_str(&s) -// } +#[derive(Debug, Deserialize, Serialize, Default)] +pub struct EqlEncryptedIndexes { + #[serde(rename = "o", skip_serializing_if = "Option::is_none")] + pub(crate) ore_index: Option>, + #[serde(rename = "m", skip_serializing_if = "Option::is_none")] + pub(crate) match_index: Option>, + #[serde(rename = "u", skip_serializing_if = "Option::is_none")] + pub(crate) unique_index: Option, + + #[serde(rename = "s", skip_serializing_if = "Option::is_none")] + pub(crate) selector: Option, + + #[serde(rename = "b", skip_serializing_if = "Option::is_none")] + pub(crate) blake3_index: Option, + + #[serde(rename = "ocf", skip_serializing_if = "Option::is_none")] + pub(crate) ore_cclw_fixed_index: Option, + #[serde(rename = "ocv", skip_serializing_if = "Option::is_none")] + pub(crate) ore_cclw_var_index: Option, + + #[serde(rename = "sv", skip_serializing_if = "Option::is_none")] + pub(crate) ste_vec_index: Option>, +} #[cfg(test)] mod tests { + use crate::{ + eql::{EqlEncryptedBody, EqlEncryptedIndexes}, + EqlEncrypted, + }; + use super::{Identifier, Plaintext}; - use crate::Encrypted; use cipherstash_client::zerokms::EncryptedRecord; use recipher::key::Iv; use uuid::Uuid; @@ -141,20 +145,29 @@ mod tests { pub fn ciphertext_json() { let expected = Identifier::new("table", "column"); - let ct = Encrypted::Ciphertext { + let ct = EqlEncrypted { identifier: expected.clone(), version: 1, - ciphertext: EncryptedRecord { - iv: Iv::default(), - ciphertext: vec![1; 32], - tag: vec![1; 16], - descriptor: "ciphertext".to_string(), - dataset_id: Some(Uuid::new_v4()), + body: EqlEncryptedBody { + ciphertext: EncryptedRecord { + iv: Iv::default(), + ciphertext: vec![1; 32], + tag: vec![1; 16], + descriptor: "ciphertext".to_string(), + dataset_id: Some(Uuid::new_v4()), + }, + indexes: EqlEncryptedIndexes { + ore_index: None, + match_index: None, + unique_index: None, + blake3_index: None, + selector: None, + ore_cclw_fixed_index: None, + ore_cclw_var_index: None, + ste_vec_index: None, + }, + is_array_item: None, }, - - ore_index: None, - match_index: None, - unique_index: None, }; let value = serde_json::to_value(&ct).unwrap(); @@ -163,12 +176,7 @@ mod tests { let t = &i["t"]; assert_eq!(t, "table"); - let result: Encrypted = serde_json::from_value(value).unwrap(); - - if let Encrypted::Ciphertext { identifier, .. } = result { - assert_eq!(expected, identifier); - } else { - panic!("Expected Encrypted::Ciphertext"); - } + let result: EqlEncrypted = serde_json::from_value(value).unwrap(); + assert_eq!(expected, result.identifier); } } diff --git a/packages/cipherstash-proxy/src/error.rs b/packages/cipherstash-proxy/src/error.rs index a37e7413..96621f35 100644 --- a/packages/cipherstash-proxy/src/error.rs +++ b/packages/cipherstash-proxy/src/error.rs @@ -104,7 +104,7 @@ pub enum ConfigError { Certificate(#[from] rustls_pki_types::pem::Error), #[error(transparent)] - EncryptConfig(#[from] cipherstash_config::errors::ConfigError), + EncryptConfig(#[from] cipherstash_client::config::errors::ConfigError), #[error(transparent)] Database(#[from] tokio_postgres::Error), @@ -202,9 +202,21 @@ pub enum EncryptError { #[error(transparent)] CiphertextCouldNotBeSerialised(#[from] serde_json::Error), + #[error("Encrypted column could not be parsed")] + ColumnCouldNotBeParsed, + + #[error("Encrypted column is null")] + ColumnIsNull, + + #[error("Column '{column}' in table '{table}' could not be deserialised. For help visit {}#encrypt-column-could-not-be-deserialised", ERROR_DOC_BASE_URL)] + ColumnCouldNotBeDeserialised { table: String, column: String }, + #[error("Column '{column}' in table '{table}' could not be encrypted. For help visit {}#encrypt-column-could-not-be-encrypted", ERROR_DOC_BASE_URL)] ColumnCouldNotBeEncrypted { table: String, column: String }, + #[error("Column configuration for column '{column}' in table '{table}' does not match the encrypted column. For help visit {}#encrypt-column-config-mismatch", ERROR_DOC_BASE_URL)] + ColumnConfigurationMismatch { table: String, column: String }, + /// This should in practice be unreachable #[error("Missing encrypt configuration for column type `{plaintext_type}`. For help visit {}#encrypt-missing-encrypt-configuration", ERROR_DOC_BASE_URL)] MissingEncryptConfiguration { plaintext_type: String }, @@ -285,12 +297,6 @@ impl From for Error { } } -impl From for Error { - fn from(e: cipherstash_config::errors::ConfigError) -> Self { - Error::Config(e.into()) - } -} - impl From for Error { fn from(e: cipherstash_client::encryption::TypeParseError) -> Self { Error::Encrypt(e.into()) diff --git a/packages/cipherstash-proxy/src/lib.rs b/packages/cipherstash-proxy/src/lib.rs index 202a07e8..2e81ca9e 100644 --- a/packages/cipherstash-proxy/src/lib.rs +++ b/packages/cipherstash-proxy/src/lib.rs @@ -15,7 +15,7 @@ pub use crate::cli::Args; pub use crate::cli::Migrate; pub use crate::config::{DatabaseConfig, ServerConfig, TandemConfig, TlsConfig}; pub use crate::encrypt::Encrypt; -pub use crate::eql::{Encrypted, ForQuery, Identifier, Plaintext}; +pub use crate::eql::{EqlEncrypted, ForQuery, Identifier, Plaintext}; pub use crate::log::init; use std::mem; diff --git a/packages/cipherstash-proxy/src/log/mod.rs b/packages/cipherstash-proxy/src/log/mod.rs index 3ed05f7f..b1922ba8 100644 --- a/packages/cipherstash-proxy/src/log/mod.rs +++ b/packages/cipherstash-proxy/src/log/mod.rs @@ -18,6 +18,7 @@ pub const AUTHENTICATION: &str = "authentication"; pub const CONFIG: &str = "config"; pub const CONTEXT: &str = "context"; pub const ENCRYPT: &str = "encrypt"; +pub const DECRYPT: &str = "decrypt"; pub const ENCODING: &str = "encoding"; pub const ENCRYPT_CONFIG: &str = "encrypt_config"; pub const KEYSET: &str = "keyset"; @@ -128,6 +129,7 @@ mod tests { encoding_level: LogLevel::Error, encrypt_level: LogLevel::Error, encrypt_config_level: LogLevel::Error, + decrypt_level: LogLevel::Error, keyset_level: LogLevel::Trace, migrate_level: LogLevel::Trace, protocol_level: LogLevel::Info, diff --git a/packages/cipherstash-proxy/src/log/subscriber.rs b/packages/cipherstash-proxy/src/log/subscriber.rs index a69beac9..eb1d6912 100644 --- a/packages/cipherstash-proxy/src/log/subscriber.rs +++ b/packages/cipherstash-proxy/src/log/subscriber.rs @@ -6,11 +6,14 @@ use tracing_subscriber::fmt::writer::BoxMakeWriter; use tracing_subscriber::fmt::SubscriberBuilder; use tracing_subscriber::FmtSubscriber; +use super::DECRYPT; + fn log_targets() -> Vec<&'static str> { vec![ DEVELOPMENT, AUTHENTICATION, CONTEXT, + DECRYPT, ENCRYPT, KEYSET, PROTOCOL, @@ -24,6 +27,7 @@ fn log_level_for(config: &LogConfig, target: &str) -> LogLevel { DEVELOPMENT => config.development_level, AUTHENTICATION => config.authentication_level, CONTEXT => config.context_level, + DECRYPT => config.decrypt_level, ENCRYPT => config.encrypt_level, KEYSET => config.keyset_level, PROTOCOL => config.protocol_level, diff --git a/packages/cipherstash-proxy/src/postgresql/backend.rs b/packages/cipherstash-proxy/src/postgresql/backend.rs index 76b2e74b..7a60466d 100644 --- a/packages/cipherstash-proxy/src/postgresql/backend.rs +++ b/packages/cipherstash-proxy/src/postgresql/backend.rs @@ -4,10 +4,11 @@ use super::message_buffer::MessageBuffer; use super::messages::error_response::ErrorResponse; use super::messages::row_description::RowDescription; use super::messages::BackendCode; +use super::Column; use crate::connect::Sender; use crate::encrypt::Encrypt; -use crate::eql::Encrypted; -use crate::error::Error; +use crate::eql::EqlEncrypted; +use crate::error::{EncryptError, Error}; use crate::log::{DEVELOPMENT, MAPPER, PROTOCOL}; use crate::postgresql::context::Portal; use crate::postgresql::messages::data_row::DataRow; @@ -19,7 +20,6 @@ use crate::prometheus::{ ROWS_PASSTHROUGH_TOTAL, ROWS_TOTAL, SERVER_BYTES_RECEIVED_TOTAL, }; use bytes::BytesMut; -use itertools::Itertools; use metrics::{counter, histogram}; use std::time::Instant; use tokio::io::AsyncRead; @@ -234,7 +234,7 @@ where let portal = self.context.get_portal_from_execute(); let portal = match portal.as_deref() { - Some(Portal::Encrypted { .. }) | Some(Portal::EncryptedText) => portal.unwrap(), + Some(Portal::Encrypted { .. }) => portal.unwrap(), _ => { debug!(target: MAPPER, client_id = self.context.client_id, msg = "Passthrough portal"); if !self.buffer.is_empty() { @@ -247,7 +247,7 @@ where } }; - let rows: Vec = self.buffer.drain().into_iter().collect(); + let mut rows: Vec = self.buffer.drain().into_iter().collect(); debug!(target: DEVELOPMENT, client_id = self.context.client_id, rows = rows.len()); let result_column_count = match rows.first() { @@ -261,15 +261,18 @@ where // If no portal, assume Text for all columns let result_column_format_codes = portal.format_codes(result_column_count); + let projection_columns = portal.projection_columns(); + // Each row is converted into Vec> - let ciphertexts: Vec> = rows - .iter() - .map(|row| row.to_ciphertext()) - .flatten_ok() - .collect::, _>>()?; + let ciphertexts: Vec> = rows + .iter_mut() + .flat_map(|row| row.as_ciphertext(projection_columns)) + .collect::>(); let start = Instant::now(); + self.check_column_config(projection_columns, &ciphertexts)?; + // Decrypt CipherText -> Plaintext let plaintexts = self.encrypt.decrypt(ciphertexts).await.inspect_err(|_| { counter!(DECRYPTION_ERROR_TOTAL).increment(1); @@ -313,6 +316,39 @@ where Ok(()) } + fn check_column_config( + &mut self, + projection_columns: &[Option], + ciphertexts: &[Option], + ) -> Result<(), Error> { + for (col, ct) in projection_columns.iter().zip(ciphertexts) { + match (col, ct) { + (Some(col), Some(ct)) => { + if col.identifier != ct.identifier { + return Err(EncryptError::ColumnConfigurationMismatch { + table: col.identifier.table.to_owned(), + column: col.identifier.column.to_owned(), + } + .into()); + } + } + // configured column with NULL ciphertext + (Some(_), None) => {} + // unconfigured column *should* have no ciphertext, + (None, None) => {} + // ciphertext with no column configuration is bad + (None, Some(ct)) => { + return Err(EncryptError::ColumnConfigurationMismatch { + table: ct.identifier.table.to_owned(), + column: ct.identifier.column.to_owned(), + } + .into()); + } + } + } + Ok(()) + } + async fn parameter_description_handler( &self, bytes: &BytesMut, @@ -381,7 +417,7 @@ where async fn data_row_handler(&mut self, bytes: &BytesMut) -> Result { counter!(ROWS_TOTAL).increment(1); match self.context.get_portal_from_execute().as_deref() { - Some(Portal::Encrypted { .. }) | Some(Portal::EncryptedText) => { + Some(Portal::Encrypted { .. }) => { debug!(target: MAPPER, client_id = self.context.client_id, msg = "Encrypted"); let data_row = DataRow::try_from(bytes)?; diff --git a/packages/cipherstash-proxy/src/postgresql/context/column.rs b/packages/cipherstash-proxy/src/postgresql/context/column.rs index 41f45e8b..20155ca8 100644 --- a/packages/cipherstash-proxy/src/postgresql/context/column.rs +++ b/packages/cipherstash-proxy/src/postgresql/context/column.rs @@ -1,4 +1,4 @@ -use cipherstash_config::{ColumnConfig, ColumnType}; +use cipherstash_client::schema::{ColumnConfig, ColumnType}; use postgres_types::Type; use crate::Identifier; diff --git a/packages/cipherstash-proxy/src/postgresql/context/mod.rs b/packages/cipherstash-proxy/src/postgresql/context/mod.rs index 4099335a..e4f03d78 100644 --- a/packages/cipherstash-proxy/src/postgresql/context/mod.rs +++ b/packages/cipherstash-proxy/src/postgresql/context/mod.rs @@ -74,20 +74,9 @@ pub enum Portal { format_codes: Vec, statement: Arc, }, - EncryptedText, Passthrough, } -/// -/// Portal is a Statement with Bound variables -/// An Execute message will execute the statement with the variables associated during a Bind. -/// -#[derive(Clone, Debug)] -pub struct EncryptedPortal { - pub format_codes: Vec, - pub statement: Arc, -} - impl Context { pub fn new(client_id: i32, schema: Arc) -> Context { Context { @@ -200,7 +189,6 @@ impl Context { match portal.as_ref() { Portal::Encrypted { statement, .. } => Some(statement.clone()), - Portal::EncryptedText => None, Portal::Passthrough => None, } } @@ -303,7 +291,18 @@ impl Queue { } impl Portal { - pub fn encrypted(statement: Arc, format_codes: Vec) -> Portal { + pub fn encrypted_with_format_codes( + statement: Arc, + format_codes: Vec, + ) -> Portal { + Portal::Encrypted { + statement, + format_codes, + } + } + + pub fn encrypted(statement: Arc) -> Portal { + let format_codes = vec![]; Portal::Encrypted { statement, format_codes, @@ -314,8 +313,12 @@ impl Portal { Portal::Passthrough } - pub fn encrypted_text() -> Portal { - Portal::EncryptedText + pub fn projection_columns(&self) -> &Vec> { + static EMPTY: Vec> = vec![]; + match self { + Portal::Encrypted { statement, .. } => &statement.projection_columns, + _ => &EMPTY, + } } // FormatCodes should not be None at this point @@ -336,7 +339,6 @@ impl Portal { } _ => format_codes.clone(), }, - Portal::EncryptedText => vec![FormatCode::Text; row_len], Portal::Passthrough => { unreachable!() } @@ -365,7 +367,7 @@ mod tests { } fn portal(statement: &Arc) -> Portal { - Portal::encrypted(statement.clone(), vec![]) + Portal::encrypted_with_format_codes(statement.clone(), vec![]) } fn get_statement(portal: Arc) -> Arc { diff --git a/packages/cipherstash-proxy/src/postgresql/data/from_sql.rs b/packages/cipherstash-proxy/src/postgresql/data/from_sql.rs index 24daad7d..7850741f 100644 --- a/packages/cipherstash-proxy/src/postgresql/data/from_sql.rs +++ b/packages/cipherstash-proxy/src/postgresql/data/from_sql.rs @@ -6,8 +6,7 @@ use crate::{ use bigdecimal::BigDecimal; use bytes::BytesMut; use chrono::NaiveDate; -use cipherstash_client::encryption::Plaintext; -use cipherstash_config::ColumnType; +use cipherstash_client::{encryption::Plaintext, schema::ColumnType}; use postgres_types::FromSql; use postgres_types::Type; use rust_decimal::Decimal; @@ -342,8 +341,10 @@ mod tests { }; use bytes::{BufMut, BytesMut}; use chrono::NaiveDate; - use cipherstash_client::encryption::Plaintext; - use cipherstash_config::{ColumnConfig, ColumnMode, ColumnType}; + use cipherstash_client::{ + encryption::Plaintext, + schema::{ColumnConfig, ColumnMode, ColumnType}, + }; use postgres_types::{ToSql, Type}; fn to_message(s: &[u8]) -> BytesMut { diff --git a/packages/cipherstash-proxy/src/postgresql/frontend.rs b/packages/cipherstash-proxy/src/postgresql/frontend.rs index 681b7f50..da272e19 100644 --- a/packages/cipherstash-proxy/src/postgresql/frontend.rs +++ b/packages/cipherstash-proxy/src/postgresql/frontend.rs @@ -23,7 +23,7 @@ use crate::prometheus::{ STATEMENTS_ENCRYPTED_TOTAL, STATEMENTS_PASSTHROUGH_TOTAL, STATEMENTS_TOTAL, STATEMENTS_UNMAPPABLE_TOTAL, }; -use crate::Encrypted; +use crate::EqlEncrypted; use bytes::BytesMut; use cipherstash_client::encryption::Plaintext; use eql_mapper::{self, EqlMapperError, EqlValue, TableColumn, TypeCheckedStatement}; @@ -35,6 +35,7 @@ use sqltk::parser::dialect::PostgreSqlDialect; use sqltk::parser::parser::Parser; use sqltk::NodeKey; use std::collections::HashMap; +use std::sync::Arc; use std::time::Instant; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use tracing::{debug, error, warn}; @@ -311,7 +312,7 @@ where counter!(STATEMENTS_ENCRYPTED_TOTAL).increment(1); // Set Encrypted portal - portal = Portal::encrypted_text(); + portal = Portal::encrypted(Arc::new(statement)); } None => { debug!(target: MAPPER, @@ -359,7 +360,7 @@ where &mut self, typed_statement: &TypeCheckedStatement<'_>, literal_columns: &Vec>, - ) -> Result>, Error> { + ) -> Result>, Error> { let literal_values = typed_statement.literal_values(); if literal_values.is_empty() { debug!(target: MAPPER, @@ -404,7 +405,7 @@ where async fn transform_statement( &mut self, typed_statement: &TypeCheckedStatement<'_>, - encrypted_literals: &Vec>, + encrypted_literals: &Vec>, ) -> Result, Error> { // Convert literals to ast Expr let mut encrypted_expressions = vec![]; @@ -671,7 +672,10 @@ where bind.rewrite(encrypted)?; } if statement.has_projection() { - portal = Portal::encrypted(statement, bind.result_columns_format_codes.to_owned()); + portal = Portal::encrypted_with_format_codes( + statement, + bind.result_columns_format_codes.to_owned(), + ); } }; @@ -704,7 +708,7 @@ where &mut self, bind: &Bind, statement: &Statement, - ) -> Result>, Error> { + ) -> Result>, Error> { let plaintexts = bind.to_plaintext(&statement.param_columns, &statement.postgres_param_types)?; @@ -890,15 +894,11 @@ where msg = "Configured column not found. Encryption configuration may have been deleted.", ?identifier, ); - if self.encrypt.config.mapping_errors_enabled() { - Err(EncryptError::UnknownColumn { - table: identifier.table.to_owned(), - column: identifier.column.to_owned(), - } - .into()) - } else { - Ok(None) + Err(EncryptError::UnknownColumn { + table: identifier.table.to_owned(), + column: identifier.column.to_owned(), } + .into()) } } } diff --git a/packages/cipherstash-proxy/src/postgresql/handler.rs b/packages/cipherstash-proxy/src/postgresql/handler.rs index b1acd30e..711e92e2 100644 --- a/packages/cipherstash-proxy/src/postgresql/handler.rs +++ b/packages/cipherstash-proxy/src/postgresql/handler.rs @@ -84,7 +84,7 @@ pub async fn handler( AsyncStream::Tcp(stream) => { // The Client is connecting to our Server let tls_stream = tls::server(stream, tls).await?; - client_stream = AsyncStream::Tls(tls_stream); + client_stream = AsyncStream::Tls(Box::new(tls_stream)); } AsyncStream::Tls(_) => { unreachable!(); diff --git a/packages/cipherstash-proxy/src/postgresql/messages/authentication/auth.rs b/packages/cipherstash-proxy/src/postgresql/messages/authentication/auth.rs index 25f98da1..e87a4849 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/authentication/auth.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/authentication/auth.rs @@ -56,9 +56,9 @@ impl Authentication { pub fn is_scram_sha_256_plus(&self) -> bool { match self.method { - AuthenticationMethod::Sasl { ref mechanisms } => mechanisms - .iter() - .any(|m| *m == SaslMechanism::ScramSha256Plus), + AuthenticationMethod::Sasl { ref mechanisms } => { + mechanisms.contains(&SaslMechanism::ScramSha256Plus) + } _ => false, } } diff --git a/packages/cipherstash-proxy/src/postgresql/messages/bind.rs b/packages/cipherstash-proxy/src/postgresql/messages/bind.rs index e259b3b6..a76a02f6 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/bind.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/bind.rs @@ -76,7 +76,7 @@ impl Bind { Ok(plaintexts) } - pub fn rewrite(&mut self, encrypted: Vec>) -> Result<(), Error> { + pub fn rewrite(&mut self, encrypted: Vec>) -> Result<(), Error> { for (idx, ct) in encrypted.iter().enumerate() { if let Some(ct) = ct { let json = serde_json::to_value(ct)?; diff --git a/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs b/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs index e12274fe..5190d911 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs @@ -1,12 +1,13 @@ -use super::{maybe_json, maybe_jsonb, BackendCode, NULL}; +use super::{BackendCode, NULL}; use crate::{ eql, - error::{Error, ProtocolError}, - log::MAPPER, + error::{EncryptError, Error, ProtocolError}, + log::DECRYPT, + postgresql::Column, }; use bytes::{Buf, BufMut, BytesMut}; use std::io::Cursor; -use tracing::debug; +use tracing::{debug, error}; #[derive(Debug, Clone)] pub struct DataRow { @@ -19,8 +20,37 @@ pub struct DataColumn { } impl DataRow { - pub fn to_ciphertext(&self) -> Result>, Error> { - Ok(self.columns.iter().map(|col| col.into()).collect()) + pub fn as_ciphertext( + &mut self, + column_configuration: &Vec>, + ) -> Vec> { + let mut result = vec![]; + for (data_column, column_config) in self.columns.iter_mut().zip(column_configuration) { + let encrypted = column_config + .as_ref() + .filter(|_| data_column.is_not_null()) + .and_then(|config| { + data_column + .try_into() + .inspect_err(|err| match err { + Error::Encrypt(EncryptError::ColumnIsNull) => { + // Not an error, as you were + data_column.set_null(); + } + _ => { + let err = EncryptError::ColumnCouldNotBeDeserialised { + table: config.identifier.table.to_owned(), + column: config.identifier.column.to_owned(), + }; + error!(target: DECRYPT, msg = err.to_string()); + } + }) + .ok() + }); + result.push(encrypted); + } + + result } pub fn column_count(&self) -> usize { @@ -47,14 +77,12 @@ impl DataRow { } impl DataColumn { - pub fn get_data(&self) -> Option> { - self.bytes.as_ref().map(|b| b.to_vec()) + pub fn is_not_null(&self) -> bool { + self.bytes.is_some() } - pub fn maybe_ciphertext(&self) -> bool { - self.bytes - .as_ref() - .is_some_and(|b| maybe_jsonb(b) || maybe_json(b)) + pub fn set_null(&mut self) { + self.bytes = None; } pub fn rewrite(&mut self, b: &[u8]) { @@ -63,21 +91,6 @@ impl DataColumn { bytes.extend_from_slice(b); } } - - /// - /// If the json format looks binary, returns a reference to the bytes without the jsonb header byte - /// - pub fn json_bytes(&self) -> Option<&[u8]> { - self.bytes.as_ref().and_then(|b| { - if maybe_jsonb(b) { - Some(&b[1..]) - } else if maybe_json(b) { - Some(&b[0..]) - } else { - None - } - }) - } } impl TryFrom<&BytesMut> for DataRow { @@ -108,9 +121,11 @@ impl TryFrom<&BytesMut> for DataRow { columns.push(DataColumn { bytes: None }); } else { let len = len as usize; + let mut bytes = BytesMut::with_capacity(len); bytes.resize(len, 0); cursor.copy_to_slice(&mut bytes); + columns.push(DataColumn { bytes: Some(bytes) }); } } @@ -159,108 +174,224 @@ impl TryFrom for BytesMut { } } -impl From<&DataColumn> for Option { - fn from(col: &DataColumn) -> Self { - debug!(target: MAPPER, data_column = ?col); - match col.json_bytes() { - Some(bytes) => match serde_json::from_slice(bytes) { - Ok(ct) => Some(ct), - Err(err) => { - debug!(target: MAPPER, msg = "Could not convert DataColumn to Ciphertext", error = err.to_string()); - None +impl TryFrom<&mut DataColumn> for eql::EqlEncrypted { + type Error = Error; + + fn try_from(col: &mut DataColumn) -> Result { + if let Some(bytes) = &col.bytes { + if &bytes[0..=1] == b"(\"" { + // Text encoding + // Encrypted record is in the form ("{}") + // json data can be extracted by dropping the first and last two bytes to remove (" and ") + let start = 2; + let end = bytes.len() - 2; + let sliced = &bytes[start..end]; + + let input = String::from_utf8_lossy(sliced).to_string(); + let input = input.replace("\"\"", "\""); + + match serde_json::from_str(&input) { + Ok(e) => return Ok(e), + Err(err) => { + debug!(target: DECRYPT, error = err.to_string()); + return Err(err.into()); + } } - }, - None => None, + } else { + // 12 bytes for the binary rowtype header + // plus 1 byte for the jsonb header (value of 1) + // [Int32] Number of fields (N) + // [Int32] OID of the field’s type + // [Int32] Length of the field (in bytes), or -1 for NULL + + let start = 4 + 4; + let end = 4 + 4 + 4; + + let mut len_bytes = [0u8; 4]; // Create a fixed-size array + len_bytes.copy_from_slice(&bytes[start..end]); + + let len = i32::from_be_bytes(len_bytes); + + if len == NULL { + return Err(EncryptError::ColumnIsNull.into()); + } + + let start = 12 + 1; + let sliced = &bytes[start..]; + + match serde_json::from_slice(sliced) { + Ok(e) => return Ok(e), + Err(err) => { + debug!(target: DECRYPT, error = err.to_string()); + return Err(err.into()); + } + } + } } + + Err(EncryptError::ColumnCouldNotBeParsed.into()) } } #[cfg(test)] mod tests { use super::DataRow; - use crate::{config::LogConfig, log, postgresql::messages::data_row::DataColumn}; - use bytes::{Buf, BytesMut}; - use cipherstash_client::zerokms::EncryptedRecord; - use recipher::key::Iv; - use tracing::info; - use uuid::Uuid; + use crate::Identifier; + use crate::{ + config::{LogConfig, LogLevel}, + log, + postgresql::{messages::data_row::DataColumn, Column}, + }; + use bytes::BytesMut; + use cipherstash_client::schema::{ColumnConfig, ColumnType}; fn to_message(s: &[u8]) -> BytesMut { BytesMut::from(s) } - fn record() -> EncryptedRecord { - EncryptedRecord { - iv: Iv::default(), - ciphertext: vec![1; 32], - tag: vec![1; 16], - descriptor: "users/name".to_string(), - dataset_id: Some(Uuid::new_v4()), - } + fn column_config(column: &str) -> Option { + let identifier = Identifier::new("encrypted", column); + let config = ColumnConfig::build("column".to_string()).casts_as(ColumnType::SmallInt); + let column = Column::new(identifier, config); + Some(column) + } + + fn column_config_with_id(column: &str) -> Vec> { + vec![None, column_config(column)] } #[test] - pub fn data_row_to_ciphertext() { - log::init(LogConfig::default()); + pub fn to_ciphertext_with_binary_encoding() { + log::init(LogConfig::with_level(LogLevel::Debug)); + + // Binary + // SELECT id, encrypted_text FROM encrypted WHERE id = $1 + let bytes = to_message(b"D\0\0\nR\0\x02\0\0\0\x08w\xaam\xf8Y$\x9dI\0\0\n<\0\0\0\x01\0\0\x0e\xda\0\0\n0\x01{\"b\": null, \"c\": \"mBbLbP2ww9ymEpm_yfj>@=^)JCqtLxcewai)Ilzx#HbC2p3F;dB`XP9af|s-igMjdMWLYPqYWAB#2|%mNt9E;-h3xf8wDrq~v|IvQ=jXYG!u4Uu9SI)@Q+xmSd+PWo=<;Y$Ct\",\"k\": \"ct\",\"i\": {\"t\": \"\"users\"\",\"c\": \"\"email\"\"},\"v\": 1}"; + // Two rows + assert!(encrypted[0].is_none()); + assert!(encrypted[1].is_some()); + + assert_eq!( + column_config[1].as_ref().unwrap().identifier, + encrypted[1].as_ref().unwrap().identifier + ); + } + + #[test] + pub fn to_ciphertext_with_binary_encoding_and_null() { + log::init(LogConfig::with_level(LogLevel::Debug)); - let bytes = to_message(b"D\0\0\0i\0\x01\0\0\0_\x01{\"c\": \"mBbKx=EbyVyx>mNt9E;-h3xf8wDrq~v|IvQ=jXYG!u4Uu9SI)@Q+xmSd+PWo=<;Y$Ct\",\"k\": \"ct\",\"i\": {\"t\": \"\"users\"\",\"c\": \"\"email\"\"},\"v\": 1}"); - // let expected = bytes.clone(); + // Binary + // encrypted_text IS NULL + // SELECT id, encrypted_text FROM encrypted WHERE id = $1 - let _data_row = DataRow::try_from(&bytes).unwrap(); + // let bytes = to_message(b"D\0\0\0\"\0\x02\0\0\0\x089\"\x88A\xe59\xb0\x13\0\0\0\x0c\0\0\0\x01\0\0\x0e\xda\xff\xff\xff\xff"); + let bytes = to_message(b"D\0\0\0\"\0\x02\0\0\0\x08>\xe6=uOUiFLgDpZXhU#s#%c4wyi&Z7`(d0IxUty-cI#Yp%o~QFF39^sRf>4*EG{zlk;}ArEQ}NQHa9@;T73aPOSTpuh\"\", \"\"i\"\": {\"\"c\"\": \"\"encrypted_jsonb\"\", \"\"t\"\": \"\"encrypted\"\"}, \"\"m\"\": null, \"\"o\"\": null, \"\"s\"\": null, \"\"u\"\": null, \"\"v\"\": 1, \"\"sv\"\": [{\"\"b\"\": \"\"8067db44a848ab32c3056a3dbe4edf16\"\", \"\"c\"\": \"\"mBbLR(BvRN1BF^PAFs!B^`U;mA>uOUiFLgDpZXhU#s#%c4wyi&Z7`(d0IxUty-cI#Yp%o~QFF39^sRf>4*EG{zlk;}ArEQ}NQHa9@;T73aPOSTpuh\"\", \"\"m\"\": null, \"\"o\"\": null, \"\"s\"\": \"\"9493d6010fe7845d52149b697729c745\"\", \"\"u\"\": null, \"\"sv\"\": null, \"\"ocf\"\": null, \"\"ocv\"\": null}, {\"\"b\"\": null, \"\"c\"\": \"\"mBbLR(BvRN1BF^PAFs!B^`U;m8QkTKr|h>Q`^NbW(CC|>SD}UM=o%mz(Fw#LQFF39^sRf>4*EG{zlk;}ArEQ}NQHa9@;T73aPOSTpuh\"\", \"\"m\"\": null, \"\"o\"\": null, \"\"s\"\": \"\"b1f0e4bb3855bc33936ef1fddf532765\"\", \"\"u\"\": null, \"\"sv\"\": null, \"\"ocf\"\": null, \"\"ocv\"\": \"\"fbc7a11fc81f2a31c904c5b05572b054824e3b5f5ece78f1b711f93175f0a4a9726157cea247e107\"\"}], \"\"ocf\"\": null, \"\"ocv\"\": null}\")"); + let mut data_row = DataRow::try_from(&bytes).unwrap(); - // let ciphertext = data_row.to_ciphertext().expect("ok"); + assert!(data_row.columns[0].bytes.is_some()); - // info!("{:?}", data_row); + let column_config = vec![column_config("encrypted_jsonb")]; + let encrypted = data_row.as_ciphertext(&column_config); - // // info!("{:?}", ciphertext.first()); + assert_eq!(encrypted.len(), 1); + assert!(encrypted[0].is_some()); - // // let column = ciphertext.first().unwrap().as_ref().unwrap(); + assert_eq!( + column_config[0].as_ref().unwrap().identifier, + encrypted[0].as_ref().unwrap().identifier + ); + } - // // assert_eq!(column.kind, "ct"); - // } + #[test] + pub fn to_ciphertext_with_text_encoding_and_null() { + log::init(LogConfig::with_level(LogLevel::Debug)); + + // SELECT * FROM encrypted WHERE id = $1; + // Only encrypted_text is NOT NULL + let bytes = to_message(b"D\0\0\n\x91\0\n\0\0\0\n1297231342\xff\xff\xff\xff\0\0\nY(\"{\"\"b\"\": null, \"\"c\"\": \"\"mBbJ;S^xMu@++(U20{lxK;qYYaDYF#30N~x;wyOUMoFOB9K!>A_9g9j@+M6V3wENqu#H8gDb9OZewzJaCBv4Uvy=7bie\"\", \"\"i\"\": {\"\"c\"\": \"\"encrypted_text\"\", \"\"t\"\": \"\"encrypted\"\"}, \"\"m\"\": [369, 381, 1758, 403, 35, 609, 1181, 1098, 1347, 1633, 1150, 815, 1997, 234, 1858, 656, 1335, 936, 1204, 630, 1764, 1328, 1649, 1396, 113, 1149, 1499, 1147, 586, 1942, 901, 1256, 1226, 1045, 637, 279, 1162, 1077, 1340, 1336, 1448, 700, 176, 1849, 1915, 1389, 71, 515, 633, 388, 1877, 1339, 1239, 638, 1365, 1380, 1273, 581, 1792, 1716, 145, 512, 814, 272, 1333, 1775, 1572, 1744, 2018, 433, 1641, 1529, 647, 1317, 652, 1606, 1737, 470, 826, 80, 929, 1700, 1619, 1253, 358, 1589, 1971, 1019, 1533, 1624, 573, 1684, 1287, 575, 1761, 527, 404, 1369, 894, 18, 1101, 986, 1772, 1090, 1506, 2015, 1988, 205, 141, 445, 1982], \"\"o\"\": [\"\"faa1f63cb6d36094d1aa50db6c0217eb447a987071119bb127f677b6a7ee0b4fe40eed7cd84e96e8a11bbe3ea14331f3ec4c8f149ce9d2b0253b4676c86557fcec4a5f8ca4e1ee081c66bf0a3cb594c6b5739f77f62fc5e76991869c23a97f01816cde3dfc24b2ca2fbb12b50fde324f18aa51718d681772bf9caf3c059a6748cbcaf4dd1c4fa02645d74699d7d265faf938c339f6cc8f57db9bd4cff8e03cae9e5d21a651b33525e86e335dff61520e8f23d7002f05fa186075a335fb7b2c740133b5a72760ccd216127d69983aa31a090a3b6ca56a48b6372cab60c979465d84dc94e5452c92517b643882fa82c22a26b4feaaa1b0ae8fcb989b10d0351fb3c9c5e56e719f820442612a67fff334438f3f5d35ff6db1b5f7a50670c7fec014f6fc19c352eb011911faf62a230e10c2d16f6c84b46cf9ee7eb1afb9c61a523891e31da2a18b445769d75c11873566dc8196d77e985423226bd1db10e4ce9eb10c2f69db7ce57d47281401617978d2bcfca23b9015b9e705615b8bf773daa87a18417f86e5338a7929fa4f10c6864af09870bfd9ddfb7848\"\", \"\"b41d89a196a35252a965ce3c330eac369ead56e9f06e2016da4d6971fe0b8d6e677e1018e7a1bd2fa0b2c1faaa12650d678352ecc81f6be879213fe78b8004b87dd7dcadec59df4dcafdb3c9aa55dcb2cc2bcf2193574b201c9a1c14764d69716f63b0c1aa30a2846696f2a1c790ca2cb26370d7e20904a8748ea98a95ee3cbb95c5f342de4e71bbf0262e84d59188ea72fe4449a16e7c73f88ed06b9cb724902a85d063c03e9b1a63dd18b9604625ca3cb8110d9c8f93e1771525c51b6ee092d554e84d61df5b557994f32191bb2b6801d9727fb707d5287e6c83d6b16763a6e66526baf80765a58d36df744be7872d2750eb28a86a519a21ee710f618c09cb2bd45f21e805ae4e11eb2987d7be31c32164d4f828fc35c389d516d0d6a54e25041985cffcb6124b4d3fa5b0ba91e19d60e3102370e9c1c768df1b427c682304a1dfdea2d3e514db22057f43d8121b8daf7c434831e5b618bbca9f4e198741927bdc168e4703fb1f703957f7b70491e06bec4adee19d29ef5e938695e1d49ef50ceef0a9c3e46bd8fe309e013e5ea0d35c5ebf3dddd97573\"\"], \"\"s\"\": null, \"\"u\"\": \"\"962d77dfaf892b596b3255c022359e54f3e8dc8b21c3d1b32ebd05555f433192\"\", \"\"v\"\": 1, \"\"sv\"\": null, \"\"ocf\"\": null, \"\"ocv\"\": null}\")\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"); + + let mut data_row = DataRow::try_from(&bytes).unwrap(); + + assert!(data_row.columns[0].bytes.is_some()); + + let column_config = vec![ + None, + None, + column_config("encrypted_text"), + column_config("encrypted_bool"), + column_config("encrypted_int2"), + column_config("encrypted_int4"), + column_config("encrypted_int8"), + column_config("encrypted_float8"), + column_config("encrypted_date"), + column_config("encrypted_jsonb"), + ]; + + let encrypted = data_row.as_ciphertext(&column_config); + + assert_eq!(encrypted.len(), 10); + + assert!(encrypted[0].is_none()); + assert!(encrypted[1].is_none()); + assert!(encrypted[2].is_some()); // <-- Some + assert!(encrypted[3].is_none()); + // etc + + assert_eq!( + column_config[2].as_ref().unwrap().identifier, + encrypted[2].as_ref().unwrap().identifier + ); + } #[test] pub fn parse_data_row() { - let bytes = to_message(b"D\0\0\0\x0e\0\x01\0\0\0\x04\0\0\x1e\xa2"); - let expected = bytes.clone(); + log::init(LogConfig::with_level(LogLevel::Debug)); - let data_row = DataRow::try_from(&bytes).unwrap(); + let messages = vec![ + to_message(b"D\0\0\0\x0e\0\x01\0\0\0\x04\0\0\x1e\xa2"), + // SELECT encrypted_jsonb FROM encrypted LIMIT 1 + to_message(b"D\0\0\x03\xba\0\x01\0\0\x03\xb0(\"{\"\"b\"\": null, \"\"c\"\": \"\"mBbLR(BvRN1BF^PAFs!B^`U;mA>uOUiFLgDpZXhU#s#%c4wyi&Z7`(d0IxUty-cI#Yp%o~QFF39^sRf>4*EG{zlk;}ArEQ}NQHa9@;T73aPOSTpuh\"\", \"\"i\"\": {\"\"c\"\": \"\"encrypted_jsonb\"\", \"\"t\"\": \"\"encrypted\"\"}, \"\"m\"\": null, \"\"o\"\": null, \"\"s\"\": null, \"\"u\"\": null, \"\"v\"\": 1, \"\"sv\"\": [{\"\"b\"\": \"\"8067db44a848ab32c3056a3dbe4edf16\"\", \"\"c\"\": \"\"mBbLR(BvRN1BF^PAFs!B^`U;mA>uOUiFLgDpZXhU#s#%c4wyi&Z7`(d0IxUty-cI#Yp%o~QFF39^sRf>4*EG{zlk;}ArEQ}NQHa9@;T73aPOSTpuh\"\", \"\"m\"\": null, \"\"o\"\": null, \"\"s\"\": \"\"9493d6010fe7845d52149b697729c745\"\", \"\"u\"\": null, \"\"sv\"\": null, \"\"ocf\"\": null, \"\"ocv\"\": null}, {\"\"b\"\": null, \"\"c\"\": \"\"mBbLR(BvRN1BF^PAFs!B^`U;m8QkTKr|h>Q`^NbW(CC|>SD}UM=o%mz(Fw#LQFF39^sRf>4*EG{zlk;}ArEQ}NQHa9@;T73aPOSTpuh\"\", \"\"m\"\": null, \"\"o\"\": null, \"\"s\"\": \"\"b1f0e4bb3855bc33936ef1fddf532765\"\", \"\"u\"\": null, \"\"sv\"\": null, \"\"ocf\"\": null, \"\"ocv\"\": \"\"fbc7a11fc81f2a31c904c5b05572b054824e3b5f5ece78f1b711f93175f0a4a9726157cea247e107\"\"}], \"\"ocf\"\": null, \"\"ocv\"\": null}\")"), + ]; - let data_col = data_row.columns.first().unwrap(); + for bytes in messages { + let expected = bytes.clone(); - let mut buf: &[u8] = data_col.bytes.as_ref().unwrap(); - let value = buf.get_i32(); - assert_eq!(value, 7842); + let data_row = DataRow::try_from(&bytes).unwrap(); - let bytes = BytesMut::try_from(data_row).unwrap(); - assert_eq!(bytes, expected); + let bytes = BytesMut::try_from(data_row).unwrap(); + assert_eq!(bytes, expected); + } } #[test] diff --git a/packages/cipherstash-proxy/src/postgresql/messages/parse.rs b/packages/cipherstash-proxy/src/postgresql/messages/parse.rs index faf0ec6b..c54d0cfa 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/parse.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/parse.rs @@ -24,11 +24,11 @@ impl Parse { } /// - /// Encrypted columns are the cs_encrypted_v1 Domain Type - /// cs_encrypted_v1 wraps JSONB + /// Encrypted columns are the eql_v2_encrypted Domain Type + /// eql_v2_encrypted wraps JSONB /// - /// Using JSONB to avoid the complexity of loading the OID of cs_encrypted_v1 - /// PostgreSQL will coerce JSONB to cs_encrypted_v1 if it passes the constaint check + /// Using JSONB to avoid the complexity of loading the OID of eql_v2_encrypted + /// PostgreSQL will coerce JSONB to eql_v2_encrypted if it passes the constaint check /// pub fn rewrite_param_types(&mut self, columns: &[Option]) { for (idx, col) in columns.iter().enumerate() { @@ -123,7 +123,7 @@ mod tests { Identifier, }; use bytes::BytesMut; - use cipherstash_config::{ColumnConfig, ColumnType}; + use cipherstash_client::schema::{ColumnConfig, ColumnType}; fn to_message(s: &[u8]) -> BytesMut { BytesMut::from(s) diff --git a/packages/cipherstash-proxy/src/postgresql/startup.rs b/packages/cipherstash-proxy/src/postgresql/startup.rs index e079b911..dd45bb31 100644 --- a/packages/cipherstash-proxy/src/postgresql/startup.rs +++ b/packages/cipherstash-proxy/src/postgresql/startup.rs @@ -30,7 +30,7 @@ pub async fn with_tls(stream: AsyncStream, config: &TandemConfig) -> Result { let tls_stream = tls::client(tcp_stream, config).await?; - Ok(AsyncStream::Tls(tls_stream)) + Ok(AsyncStream::Tls(Box::new(tls_stream))) } false => { warn!(msg = "Connecting to database without Transport Layer Security (TLS)"); diff --git a/packages/eql-mapper/Cargo.toml b/packages/eql-mapper/Cargo.toml index febb293e..3c63989e 100644 --- a/packages/eql-mapper/Cargo.toml +++ b/packages/eql-mapper/Cargo.toml @@ -19,6 +19,7 @@ sqltk = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } +vec1 = "1.12.1" [dev-dependencies] pretty_assertions = "^1.0" diff --git a/packages/eql-mapper/src/inference/infer_type_impls/expr.rs b/packages/eql-mapper/src/inference/infer_type_impls/expr.rs index 667d4dc7..51c03c12 100644 --- a/packages/eql-mapper/src/inference/infer_type_impls/expr.rs +++ b/packages/eql-mapper/src/inference/infer_type_impls/expr.rs @@ -24,12 +24,10 @@ impl<'ast> InferType<'ast, Expr> for TypeInferencer<'ast> { self.unify_node_with_type(this_expr, self.resolve_compound_ident(idents)?)?; } - #[allow(unused_variables)] Expr::Wildcard(_) => { self.unify_node_with_type(this_expr, self.resolve_wildcard()?)?; } - #[allow(unused_variables)] Expr::QualifiedWildcard(object_name, _) => { self.unify_node_with_type( this_expr, @@ -158,7 +156,7 @@ impl<'ast> InferType<'ast, Expr> for TypeInferencer<'ast> { | BinaryOperator::HashArrow | BinaryOperator::HashLongArrow | BinaryOperator::AtAt - | BinaryOperator::HashMinus + | BinaryOperator::HashMinus // TODO do not support for EQL | BinaryOperator::AtQuestion | BinaryOperator::Question | BinaryOperator::QuestionAnd diff --git a/packages/eql-mapper/src/inference/infer_type_impls/function.rs b/packages/eql-mapper/src/inference/infer_type_impls/function.rs index 88347df6..515bcc83 100644 --- a/packages/eql-mapper/src/inference/infer_type_impls/function.rs +++ b/packages/eql-mapper/src/inference/infer_type_impls/function.rs @@ -1,12 +1,16 @@ use eql_mapper_macros::trace_infer; -use sqltk::parser::ast::{Function, FunctionArg, FunctionArgExpr, FunctionArguments, Ident}; +use sqltk::parser::ast::{Function, FunctionArguments}; use crate::{ - inference::{type_error::TypeError, InferType}, - unifier::Type, - SqlIdent, TypeInferencer, + get_sql_function_def, inference::infer_type::InferType, CompoundIdent, FunctionSig, TypeError, + TypeInferencer, }; +/// Looks up the function signature. +/// +/// If a signature is found it means that function is handled as an EQL special case and is type checked accordingly. +/// +/// If a signature is not found then all function args and its return type are unified as native. #[trace_infer] impl<'ast> InferType<'ast, Function> for TypeInferencer<'ast> { fn infer_exit(&mut self, function: &'ast Function) -> Result<(), TypeError> { @@ -17,115 +21,17 @@ impl<'ast> InferType<'ast, Function> for TypeInferencer<'ast> { } let Function { name, args, .. } = function; - - let fn_name: Vec<_> = name.0.iter().map(SqlIdent).collect(); - - if fn_name == [SqlIdent(&Ident::new("min"))] || fn_name == [SqlIdent(&Ident::new("max"))] { - // 1. There MUST be one unnamed argument (it CAN come from a subquery) - // 2. The return type is the same as the argument type - - match args { - FunctionArguments::None => { - return Err(TypeError::FunctionCall(format!( - "{} should be called with 1 argument, got 0", - fn_name.last().unwrap() - ))) - } - - FunctionArguments::Subquery(query) => { - // The query must return a single column projection which has the same type as the result of the - // call to min/max. - self.unify_node_with_type( - &**query, - Type::projection(&[(self.get_node_type(function), None)]), - )?; - } - - FunctionArguments::List(args_list) => { - if args_list.args.len() == 1 { - match &args_list.args[0] { - FunctionArg::Named { .. } | FunctionArg::ExprNamed { .. } => { - return Err(TypeError::FunctionCall(format!( - "{} cannot be called with named arguments", - fn_name.last().unwrap(), - ))) - } - - FunctionArg::Unnamed(function_arg_expr) => match function_arg_expr { - FunctionArgExpr::Expr(expr) => { - self.unify_nodes(function, expr)?; - } - - FunctionArgExpr::QualifiedWildcard(_) - | FunctionArgExpr::Wildcard => { - return Err(TypeError::FunctionCall(format!( - "{} cannot be called with wildcard arguments", - fn_name.last().unwrap(), - ))) - } - }, - } - } else { - return Err(TypeError::FunctionCall(format!( - "{} should be called with 1 argument, got {}", - fn_name.last().unwrap(), - args_list.args.len() - ))); - } - } + let fn_name = CompoundIdent::from(&name.0); + + match get_sql_function_def(&fn_name, args) { + Some(sql_fn) => { + sql_fn + .sig + .instantiate(&*self) + .apply_constraints(self, function)?; } - } else { - // All other functions: resolve to native - // EQL values will be rejected in function calls - self.unify_node_with_type(function, Type::any_native())?; - - match args { - // Function called without any arguments. - // Used for functions like `CURRENT_TIMESTAMP` that do not require parentheses () - // This is not the same as a function that has zero arguments (which would be an empty arg list) - FunctionArguments::None => {} - - FunctionArguments::Subquery(query) => { - // The query must return a single column projection which has the same type as the result of the function - self.unify_node_with_type( - &**query, - Type::projection(&[(self.get_node_type(function), None)]), - )?; - } - - FunctionArguments::List(args_list) => { - self.unify_node_with_type(function, Type::any_native())?; - for arg in &args_list.args { - match arg { - FunctionArg::ExprNamed { - name, - arg, - operator: _, - } => { - self.unify_node_with_type(name, Type::any_native())?; - match arg { - FunctionArgExpr::Expr(expr) => { - self.unify_node_with_type(expr, Type::any_native())?; - } - // Aggregate functions like COUNT(table.*) - FunctionArgExpr::QualifiedWildcard(_) => {} - // Aggregate functions like COUNT(*) - FunctionArgExpr::Wildcard => {} - } - } - FunctionArg::Named { arg, .. } | FunctionArg::Unnamed(arg) => match arg - { - FunctionArgExpr::Expr(expr) => { - self.unify_node_with_type(expr, Type::any_native())?; - } - // Aggregate functions like COUNT(table.*) - FunctionArgExpr::QualifiedWildcard(_) => {} - // Aggregate functions like COUNT(*) - FunctionArgExpr::Wildcard => {} - }, - } - } - } + None => { + FunctionSig::instantiate_native(function).apply_constraints(self, function)?; } } diff --git a/packages/eql-mapper/src/inference/infer_type_impls/function_arg_expr.rs b/packages/eql-mapper/src/inference/infer_type_impls/function_arg_expr.rs new file mode 100644 index 00000000..89e5d5df --- /dev/null +++ b/packages/eql-mapper/src/inference/infer_type_impls/function_arg_expr.rs @@ -0,0 +1,24 @@ +use eql_mapper_macros::trace_infer; +use sqltk::parser::ast::FunctionArgExpr; + +use crate::{inference::infer_type::InferType, TypeError, TypeInferencer}; + +#[trace_infer] +impl<'ast> InferType<'ast, FunctionArgExpr> for TypeInferencer<'ast> { + fn infer_exit(&mut self, farg_expr: &'ast FunctionArgExpr) -> Result<(), TypeError> { + let farg_expr_ty = self.get_node_type(farg_expr); + match farg_expr { + FunctionArgExpr::Expr(expr) => { + self.unify(farg_expr_ty, self.get_node_type(expr))?; + } + FunctionArgExpr::QualifiedWildcard(qualified) => { + self.unify(farg_expr_ty, self.resolve_qualified_wildcard(&qualified.0)?)?; + } + FunctionArgExpr::Wildcard => { + self.unify(farg_expr_ty, self.resolve_wildcard()?)?; + } + }; + + Ok(()) + } +} diff --git a/packages/eql-mapper/src/inference/infer_type_impls/mod.rs b/packages/eql-mapper/src/inference/infer_type_impls/mod.rs index 1f266dee..103a8cfd 100644 --- a/packages/eql-mapper/src/inference/infer_type_impls/mod.rs +++ b/packages/eql-mapper/src/inference/infer_type_impls/mod.rs @@ -1,6 +1,7 @@ // General AST nodes mod expr; mod function; +mod function_arg_expr; mod select; mod select_item; mod select_items; diff --git a/packages/eql-mapper/src/inference/mod.rs b/packages/eql-mapper/src/inference/mod.rs index 2e364944..6cb4b390 100644 --- a/packages/eql-mapper/src/inference/mod.rs +++ b/packages/eql-mapper/src/inference/mod.rs @@ -2,6 +2,8 @@ mod infer_type; mod infer_type_impls; mod registry; mod sequence; +mod sql_fn_macros; +mod sql_functions; mod type_error; pub mod unifier; @@ -12,7 +14,8 @@ use std::{cell::RefCell, fmt::Debug, marker::PhantomData, ops::ControlFlow, rc:: use infer_type::InferType; use sqltk::parser::ast::{ - Delete, Expr, Function, Ident, Insert, Query, Select, SelectItem, SetExpr, Statement, Values, + Delete, Expr, Function, FunctionArgExpr, Ident, Insert, Query, Select, SelectItem, SetExpr, + Statement, Values, }; use sqltk::{into_control_flow, AsNodeKey, Break, Visitable, Visitor}; @@ -20,6 +23,7 @@ use crate::{ScopeError, ScopeTracker, TableResolver}; pub(crate) use registry::*; pub(crate) use sequence::*; +pub(crate) use sql_functions::*; pub(crate) use type_error::*; /// [`Visitor`] implementation that performs type inference on AST nodes. @@ -123,11 +127,11 @@ impl<'ast> TypeInferencer<'ast> { match self.unify(self.get_node_type(lhs), self.get_node_type(rhs)) { Ok(unified) => Ok(unified), Err(err) => Err(TypeError::OnNodes( - Box::new(err), format!("{:?}", lhs), self.get_node_type(lhs), format!("{:?}", rhs), self.get_node_type(rhs), + err.to_string(), )), } } @@ -187,6 +191,7 @@ macro_rules! dispatch_all { dispatch!($self, $method, $node, Vec); dispatch!($self, $method, $node, SelectItem); dispatch!($self, $method, $node, Function); + dispatch!($self, $method, $node, FunctionArgExpr); dispatch!($self, $method, $node, Values); dispatch!($self, $method, $node, sqltk::parser::ast::Value); }; diff --git a/packages/eql-mapper/src/inference/sql_fn_macros.rs b/packages/eql-mapper/src/inference/sql_fn_macros.rs new file mode 100644 index 00000000..d0acec86 --- /dev/null +++ b/packages/eql-mapper/src/inference/sql_fn_macros.rs @@ -0,0 +1,55 @@ +#[macro_export] +macro_rules! to_kind { + (NATIVE) => { + $crate::Kind::Native + }; + ($generic:ident) => { + $crate::Kind::Generic(stringify!($generic)) + }; +} + +#[macro_export] +macro_rules! sql_fn_args { + (()) => { vec![] }; + + (($arg:ident)) => { vec![$crate::to_kind!($arg)] }; + + (($arg:ident $(,$rest:ident)*)) => { + vec![$crate::to_kind!($arg) $(, $crate::to_kind!($rest))*] + }; +} + +#[macro_export] +macro_rules! sql_fn { + ($name:ident $args:tt -> $return_kind:ident, rewrite) => { + $crate::SqlFunction::new( + stringify!($name), + FunctionSig::new($crate::sql_fn_args!($args), $crate::to_kind!($return_kind)), + $crate::RewriteRule::AsEqlFunction, + ) + }; + + ($name:ident $args:tt -> $return_kind:ident) => { + $crate::SqlFunction::new( + stringify!($name), + FunctionSig::new($crate::sql_fn_args!($args), $crate::to_kind!($return_kind)), + $crate::RewriteRule::Ignore, + ) + }; + + ($schema:ident . $name:ident $args:tt -> $return_kind:ident, rewrite) => { + $crate::SqlFunction::new( + stringify!($schema.$name), + FunctionSig::new($crate::sql_fn_args!($args), $crate::to_kind!($return_kind)), + $crate::RewriteRule::AsEqlFunction, + ) + }; + + ($schema:ident . $name:ident $args:tt -> $return_kind:ident) => { + $crate::SqlFunction::new( + stringify!($schema.$name), + FunctionSig::new($crate::sql_fn_args!($args), $crate::to_kind!($return_kind)), + $crate::RewriteRule::Ignore, + ) + }; +} diff --git a/packages/eql-mapper/src/inference/sql_functions.rs b/packages/eql-mapper/src/inference/sql_functions.rs new file mode 100644 index 00000000..fba4c5e2 --- /dev/null +++ b/packages/eql-mapper/src/inference/sql_functions.rs @@ -0,0 +1,266 @@ +use std::{ + collections::{HashMap, HashSet}, + sync::{Arc, LazyLock}, +}; + +use derive_more::derive::Display; +use sqltk::parser::ast::{Function, FunctionArg, FunctionArgExpr, FunctionArguments, Ident}; + +use itertools::Itertools; +use vec1::{vec1, Vec1}; + +use crate::{sql_fn, unifier::Type, SqlIdent, TypeInferencer}; + +use super::TypeError; + +/// The identifier and type signature of a SQL function. +/// +/// See [`SQL_FUNCTION_SIGNATURES`]. +#[derive(Debug)] +pub(crate) struct SqlFunction { + pub(crate) name: CompoundIdent, + pub(crate) sig: FunctionSig, + pub(crate) rewrite_rule: RewriteRule, +} + +#[derive(Debug)] +pub(crate) enum RewriteRule { + Ignore, + AsEqlFunction, +} + +/// A representation of the type of an argument or return type in a SQL function. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(crate) enum Kind { + /// A type that must be a native type + Native, + + /// A type that can be a native or EQL type. The `str` is the generic variable name. + Generic(&'static str), +} + +/// The type signature of a SQL functon (excluding its name). +#[derive(Debug, Clone)] +pub(crate) struct FunctionSig { + args: Vec, + return_type: Kind, + generics: HashSet<&'static str>, +} + +/// A function signature but filled in with fresh type variables that correspond with the [`Kind`] or each argument and +/// return type. +#[derive(Debug, Clone)] +pub(crate) struct InstantiatedSig { + args: Vec>, + return_type: Arc, +} + +impl FunctionSig { + fn new(args: Vec, return_type: Kind) -> Self { + let mut generics: HashSet<&'static str> = HashSet::new(); + + for arg in &args { + if let Kind::Generic(generic) = arg { + generics.insert(*generic); + } + } + + if let Kind::Generic(generic) = return_type { + generics.insert(generic); + } + + Self { + args, + return_type, + generics, + } + } + + /// Checks if `self` is applicable to a particular piece of SQL function invocation syntax. + pub(crate) fn is_applicable_to_args(&self, fn_args_syntax: &FunctionArguments) -> bool { + match fn_args_syntax { + FunctionArguments::None => self.args.is_empty(), + FunctionArguments::Subquery(_) => self.args.len() == 1, + FunctionArguments::List(fn_args) => self.args.len() == fn_args.args.len(), + } + } + + /// Creates an [`InstantiatedSig`] from `self`, filling in the [`Kind`]s with fresh type variables. + pub(crate) fn instantiate(&self, inferencer: &TypeInferencer<'_>) -> InstantiatedSig { + let mut generics: HashMap<&'static str, Arc> = HashMap::new(); + + for generic in self.generics.iter() { + generics.insert(generic, inferencer.fresh_tvar()); + } + + InstantiatedSig { + args: self + .args + .iter() + .map(|kind| match kind { + Kind::Native => Arc::new(Type::any_native()), + Kind::Generic(generic) => generics[generic].clone(), + }) + .collect(), + + return_type: match self.return_type { + Kind::Native => Arc::new(Type::any_native()), + Kind::Generic(generic) => generics[generic].clone(), + }, + } + } + + /// For functions that do not have special case handling we synthesise an [`InstatiatedSig`] from the SQL function + /// invocation synta where all arguments and the return types are native. + pub(crate) fn instantiate_native(function: &Function) -> InstantiatedSig { + let arg_count = match &function.args { + FunctionArguments::None => 0, + FunctionArguments::Subquery(_) => 1, + FunctionArguments::List(args) => args.args.len(), + }; + + let args: Vec> = (0..arg_count) + .map(|_| Arc::new(Type::any_native())) + .collect(); + + InstantiatedSig { + args, + return_type: Arc::new(Type::any_native()), + } + } +} + +impl InstantiatedSig { + /// Applies the type constraints of the function to to the AST. + pub(crate) fn apply_constraints<'ast>( + &self, + inferencer: &mut TypeInferencer<'ast>, + function: &'ast Function, + ) -> Result<(), TypeError> { + let fn_name = CompoundIdent::from(&function.name.0); + + inferencer.unify_node_with_type(function, self.return_type.clone())?; + + match &function.args { + FunctionArguments::None => { + if self.args.is_empty() { + Ok(()) + } else { + Err(TypeError::Conflict(format!( + "expected {} args to function {}; got 0", + self.args.len(), + fn_name + ))) + } + } + + FunctionArguments::Subquery(query) => { + if self.args.len() == 1 { + inferencer.unify_node_with_type(&**query, self.args[0].clone())?; + Ok(()) + } else { + Err(TypeError::Conflict(format!( + "expected {} args to function {}; got 0", + self.args.len(), + fn_name + ))) + } + } + + FunctionArguments::List(args) => { + for (sig_arg, fn_arg) in self.args.iter().zip(args.args.iter()) { + let farg_expr = get_function_arg_expr(fn_arg); + inferencer.unify_node_with_type(farg_expr, sig_arg.clone())?; + } + + Ok(()) + } + } + } +} + +fn get_function_arg_expr(fn_arg: &FunctionArg) -> &FunctionArgExpr { + match fn_arg { + FunctionArg::Named { arg, .. } => arg, + FunctionArg::ExprNamed { arg, .. } => arg, + FunctionArg::Unnamed(arg) => arg, + } +} + +impl SqlFunction { + fn new(ident: &str, sig: FunctionSig, rewrite_rule: RewriteRule) -> Self { + Self { + name: CompoundIdent::from(ident), + sig, + rewrite_rule, + } + } +} + +#[derive(Debug, Eq, PartialEq, PartialOrd, Ord, Hash, Clone, Display)] +#[display("{}", _0.iter().map(SqlIdent::to_string).collect::>().join("."))] +pub(crate) struct CompoundIdent(Vec1>); + +impl From<&str> for CompoundIdent { + fn from(value: &str) -> Self { + CompoundIdent(vec1![SqlIdent(Ident::new(value))]) + } +} + +impl From<&Vec> for CompoundIdent { + fn from(value: &Vec) -> Self { + let mut idents = Vec1::>::new(SqlIdent(value[0].clone())); + idents.extend(value[1..].iter().cloned().map(SqlIdent)); + CompoundIdent(idents) + } +} + +/// SQL functions that are handled with special case type checking rules. +static SQL_FUNCTIONS: LazyLock>> = LazyLock::new(|| { + // Notation: a single uppercase letter denotes an unknown type. Matching letters in a signature will be assigned + // *the same type variable* and thus must resolve to the same type. (🙏 Haskell) + // + // Eventually we should type check EQL types against their configured indexes instead of leaving that to the EQL + // extension in the database. I can imagine supporting type bounds in signatures here, such as: `T: Eq` + let sql_fns = vec![ + // TODO: when search_path support is added to the resolver we should change these + // to their fully-qualified names. + sql_fn!(count(T) -> NATIVE), + sql_fn!(min(T) -> T, rewrite), + sql_fn!(max(T) -> T, rewrite), + sql_fn!(jsonb_path_query(T, T) -> T, rewrite), + sql_fn!(jsonb_path_query_first(T, T) -> T, rewrite), + sql_fn!(jsonb_path_exists(T, T) -> T, rewrite), + sql_fn!(jsonb_array_length(T) -> T, rewrite), + sql_fn!(jsonb_array_elements(T) -> T, rewrite), + sql_fn!(jsonb_array_elements_text(T) -> T, rewrite), + // These are typings for when customer SQL already contains references to EQL functions. + // They must be type checked but not rewritten. + sql_fn!(eql_v2.min(T) -> T), + sql_fn!(eql_v2.max(T) -> T), + sql_fn!(eql_v2.jsonb_path_query(T, T) -> T), + sql_fn!(eql_v2.jsonb_path_query_first(T, T) -> T), + sql_fn!(eql_v2.jsonb_path_exists(T, T) -> T), + sql_fn!(eql_v2.jsonb_array_length(T) -> T), + sql_fn!(eql_v2.jsonb_array_elements(T) -> T), + sql_fn!(eql_v2.jsonb_array_elements_text(T) -> T), + ]; + + let mut sql_fns_by_name: HashMap> = HashMap::new(); + + for (key, chunk) in &sql_fns.into_iter().chunk_by(|sql_fn| sql_fn.name.clone()) { + sql_fns_by_name.insert(key.clone(), chunk.into_iter().collect()); + } + + sql_fns_by_name +}); + +pub(crate) fn get_sql_function_def( + fn_name: &CompoundIdent, + args: &FunctionArguments, +) -> Option<&'static SqlFunction> { + let sql_fns = SQL_FUNCTIONS.get(fn_name)?; + sql_fns + .iter() + .find(|sql_fn| sql_fn.sig.is_applicable_to_args(args)) +} diff --git a/packages/eql-mapper/src/inference/type_error.rs b/packages/eql-mapper/src/inference/type_error.rs index 2011a3d8..6b4a0c4d 100644 --- a/packages/eql-mapper/src/inference/type_error.rs +++ b/packages/eql-mapper/src/inference/type_error.rs @@ -1,4 +1,4 @@ -use std::{collections::HashSet, sync::Arc}; +use std::sync::Arc; use crate::{unifier::Type, SchemaError, ScopeError}; @@ -19,9 +19,6 @@ pub enum TypeError { #[error("{}", _0)] Expected(String), - #[error("One or more params failed to unify: {}", _0.iter().cloned().collect::>().join(", "))] - Params(HashSet), - #[error("Expected param count to be {}, but got {}", _0, _1)] ParamCount(usize, usize), @@ -34,14 +31,13 @@ pub enum TypeError { #[error("{}", _0)] SchemaError(#[from] SchemaError), - #[error("Cannot unify node types for nodes:\n 1. node: {} type: {}\n 2. node: {} type: {}\n error: {}", _1, _2, _3, _4, _0)] - OnNodes(Box, String, Arc, String, Arc), - #[error( - "Cannot unify node with type:\n node: {}\n type: {} error: {}", + "Cannot unify node types for nodes:\n 1. node: {} type: {}\n 2. node: {} type: {}\n error: {}", + _0, _1, _2, - _0 + _3, + _4 )] - OnNode(Box, Type, String), + OnNodes(String, Arc, String, Arc, String), } diff --git a/packages/eql-mapper/src/inference/unifier/mod.rs b/packages/eql-mapper/src/inference/unifier/mod.rs index e1f9e951..fb523356 100644 --- a/packages/eql-mapper/src/inference/unifier/mod.rs +++ b/packages/eql-mapper/src/inference/unifier/mod.rs @@ -447,7 +447,7 @@ pub(crate) mod test_util { } } - root_node.accept(&mut FindNodeFromKeyVisitor(self)); + let _ = root_node.accept(&mut FindNodeFromKeyVisitor(self)); } } } diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index 061ea74d..be1a0e46 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -33,6 +33,7 @@ mod test { use super::type_check; use crate::col; use crate::projection; + use crate::test_helpers; use crate::Param; use crate::Schema; use crate::TableResolver; @@ -41,9 +42,11 @@ mod test { Value, }; use pretty_assertions::assert_eq; + use sqltk::parser::ast::Ident; use sqltk::parser::ast::Statement; use sqltk::parser::ast::{self as ast}; use sqltk::AsNodeKey; + use sqltk::NodeKey; use std::collections::HashMap; use std::sync::Arc; use tracing::error; @@ -980,7 +983,7 @@ mod test { } #[test] - fn select_with_literal_subsitution() { + fn select_with_literal_cast_as_encrypted() { // init_tracing(); let schema = resolver(schema! { @@ -1017,27 +1020,63 @@ mod test { )] ); - let transformed_statement = match typed.transform(HashMap::from_iter([( + match typed.transform(HashMap::from_iter([( typed.literals[0].1.as_node_key(), ast::Value::SingleQuotedString("ENCRYPTED".into()), )])) { - Ok(transformed_statement) => transformed_statement, + Ok(transformed_statement) => assert_eq!( + transformed_statement.to_string(), + "SELECT * FROM employees WHERE salary > 'ENCRYPTED'::JSONB::eql_v2_encrypted" + ), Err(err) => panic!("statement transformation failed: {}", err), }; + } - // This type checks the transformed statement so we can get hold of the encrypted literal. - let typed = match type_check(schema, &transformed_statement) { + #[test] + fn insert_with_literal_cast_as_encrypted() { + // init_tracing(); + + let schema = resolver(schema! { + tables: { + employees: { + id, + salary (EQL), + } + } + }); + + let statement = parse( + r#" + insert into employees (salary) values (20000) + "#, + ); + + let typed = match type_check(schema.clone(), &statement) { Ok(typed) => typed, Err(err) => panic!("type check failed: {:#?}", err), }; - assert!(typed.literals.contains(&( - EqlValue(TableColumn { - table: id("employees"), - column: id("salary") - }), - &ast::Value::SingleQuotedString("ENCRYPTED".into()), - ))); + assert_eq!( + typed.literals, + vec![( + EqlValue(TableColumn { + table: id("employees"), + column: id("salary") + }), + &ast::Value::Number(20000.into(), false) + )] + ); + + match typed.transform(HashMap::from_iter([( + typed.literals[0].1.as_node_key(), + ast::Value::SingleQuotedString("ENCRYPTED".into()), + )])) { + Ok(transformed_statement) => assert_eq!( + transformed_statement.to_string(), + "INSERT INTO employees (salary) VALUES ('ENCRYPTED'::JSONB::eql_v2_encrypted)" + ), + Err(err) => panic!("statement transformation failed: {}", err), + }; } #[test] @@ -1313,7 +1352,7 @@ mod test { match typed.transform(HashMap::new()) { Ok(statement) => assert_eq!( statement.to_string(), - "SELECT CS_GROUPED_VALUE_V1(email) AS email FROM users GROUP BY CS_ORE_64_8_V1(email)".to_string() + "SELECT eql_v2.grouped_value(email) AS email FROM users GROUP BY eql_v2.ore_64_8_v2(email)".to_string() ), Err(err) => panic!("transformation failed: {err}"), } @@ -1336,14 +1375,14 @@ mod test { }); let statement = - parse("SELECT MIN(salary), MAX(salary), department FROM employees GROUP BY department"); + parse("SELECT min(salary), max(salary), department FROM employees GROUP BY department"); match type_check(schema, &statement) { Ok(typed) => { match typed.transform(HashMap::new()) { Ok(statement) => assert_eq!( statement.to_string(), - "SELECT CS_MIN_V1(salary) AS MIN, CS_MAX_V1(salary) AS MAX, department FROM employees GROUP BY department".to_string() + "SELECT eql_v2.min(salary), eql_v2.max(salary), department FROM employees GROUP BY department".to_string() ), Err(err) => panic!("transformation failed: {err}"), } @@ -1352,6 +1391,75 @@ mod test { } } + #[test] + fn select_with_params_cast_as_encrypted() { + // init_tracing(); + let schema = resolver(schema! { + tables: { + employees: { + id (PK), + eql_col (EQL), + native_col, + } + } + }); + + let statement = parse( + " + SELECT * FROM employees WHERE eql_col = $1 AND native_col = $2; + ", + ); + + match type_check(schema, &statement) { + Ok(typed) => match typed.transform(HashMap::new()) { + Ok(statement) => { + assert_eq!( + statement.to_string(), + "SELECT * FROM employees WHERE eql_col = $1::JSONB::eql_v2_encrypted AND native_col = $2" + ); + } + Err(err) => panic!("transformation failed: {err}"), + }, + Err(err) => panic!("type check failed: {err}"), + } + } + + #[test] + fn rewrite_standard_sql_fns_on_eql_types() { + // init_tracing(); + let schema = resolver(schema! { + tables: { + employees: { + id (PK), + eql_col (EQL), + native_col, + } + } + }); + + let statement = parse(" + SELECT jsonb_path_query(eql_col, '$.secret'), jsonb_path_query(native_col, '$.not-secret') FROM employees + "); + + match type_check(schema, &statement) { + Ok(typed) => { + match typed.transform(test_helpers::dummy_encrypted_json_selector( + &statement, + ast::Value::SingleQuotedString("$.secret".into()), + )) { + Ok(statement) => { + assert_eq!( + statement.to_string(), + "SELECT eql_v2.jsonb_path_query(eql_col, ''::JSONB::eql_v2_encrypted), jsonb_path_query(native_col, '$.not-secret') FROM employees" + ); + } + Err(err) => panic!("transformation failed: {err}"), + } + } + Err(err) => panic!("type check failed: {err}"), + } + } + #[test] fn supports_named_arrays() { let schema = resolver(schema! { @@ -1363,4 +1471,164 @@ mod test { type_check(schema, &statement).expect("named arrays should be supported"); } + + #[test] + fn jsonb_operator_arrow() { + test_jsonb_operator("->"); + } + + #[test] + fn jsonb_operator_long_arrow() { + test_jsonb_operator("->>"); + } + + #[test] + fn jsonb_operator_hash_arrow() { + test_jsonb_operator("#>"); + } + + #[test] + fn jsonb_operator_hash_long_arrow() { + test_jsonb_operator("#>>"); + } + + #[test] + fn jsonb_operator_hash_at_at() { + test_jsonb_operator("@@"); + } + + #[test] + fn jsonb_operator_at_question() { + test_jsonb_operator("@?"); + } + + #[test] + fn jsonb_operator_question() { + test_jsonb_operator("?"); + } + + #[test] + fn jsonb_operator_question_and() { + test_jsonb_operator("?&"); + } + + #[test] + fn jsonb_operator_question_pipe() { + test_jsonb_operator("?|"); + } + + #[test] + fn jsonb_operator_at_arrow() { + test_jsonb_operator("@>"); + } + + #[test] + fn jsonb_operator_arrow_at() { + test_jsonb_operator("<@"); + } + + #[test] + fn jsonb_function_jsonb_path_query() { + test_jsonb_function( + "jsonb_path_query", + vec![ + ast::Expr::Identifier(Ident::new("notes")), + ast::Expr::Value(ast::Value::SingleQuotedString("$.medications".to_owned())), + ], + ); + } + + // TODO: do we need to check that the RHS of JSON operators MUST be a Value node + // and not an arbitrary expression? + + fn test_jsonb_function(fn_name: &str, args: Vec) { + let schema = resolver(schema! { + tables: { + patients: { + id (PK), + notes (EQL), + } + } + }); + + let args_in = args + .iter() + .map(|expr| expr.to_string()) + .collect::>() + .join(", "); + + let statement = parse(&format!( + "SELECT id, {}({}) AS meds FROM patients", + fn_name, args_in + )); + + let args_encrypted = args + .iter() + .map(|expr| match expr { + ast::Expr::Identifier(ident) => ident.to_string(), + ast::Expr::Value(ast::Value::SingleQuotedString(s)) => { + format!("''::JSONB::eql_v2_encrypted", s) + } + _ => panic!("unsupported expr type in test util"), + }) + .collect::>() + .join(", "); + + let mut encrypted_literals: HashMap, ast::Value> = HashMap::new(); + + for arg in args.iter() { + if let ast::Expr::Value(value) = arg { + encrypted_literals.extend(test_helpers::dummy_encrypted_json_selector( + &statement, + value.clone(), + )); + } + } + + match type_check(schema, &statement) { + Ok(typed) => match typed.transform(encrypted_literals) { + Ok(statement) => { + let rewritten_fn_name = format!("eql_v2.{fn_name}"); + assert_eq!( + statement.to_string(), + format!( + "SELECT id, {}({}) AS meds FROM patients", + rewritten_fn_name, args_encrypted + ) + ) + } + Err(err) => panic!("transformation failed: {err}"), + }, + Err(err) => panic!("type check failed: {err}"), + } + } + + fn test_jsonb_operator(op: &str) { + let schema = resolver(schema! { + tables: { + patients: { + id (PK), + notes (EQL), + } + } + }); + + let statement = parse(&format!( + "SELECT id, notes {} 'medications' AS meds FROM patients", + op + )); + + match type_check(schema, &statement) { + Ok(typed) => { + match typed.transform(test_helpers::dummy_encrypted_json_selector(&statement, ast::Value::SingleQuotedString("medications".to_owned()))) { + Ok(statement) => assert_eq!( + statement.to_string(), + format!("SELECT id, notes {} ''::JSONB::eql_v2_encrypted AS meds FROM patients", op) + ), + Err(err) => panic!("transformation failed: {err}"), + } + } + Err(err) => panic!("type check failed: {err}"), + } + } } diff --git a/packages/eql-mapper/src/model/schema_delta.rs b/packages/eql-mapper/src/model/schema_delta.rs index b79c6ba9..8ae04467 100644 --- a/packages/eql-mapper/src/model/schema_delta.rs +++ b/packages/eql-mapper/src/model/schema_delta.rs @@ -196,7 +196,7 @@ pub fn collect_ddl(table_resolver: Arc, statement: &Statement) -> schema: schema_with_edits, changed: false, }; - statement.accept(&mut visitor); + let _ = statement.accept(&mut visitor); return visitor.changed; } diff --git a/packages/eql-mapper/src/model/sql_ident.rs b/packages/eql-mapper/src/model/sql_ident.rs index acaa91da..aa5a0a3b 100644 --- a/packages/eql-mapper/src/model/sql_ident.rs +++ b/packages/eql-mapper/src/model/sql_ident.rs @@ -102,14 +102,28 @@ impl SqlIdent { } } -// This manual Hash implementation is required to prevent a clippy error: -// "error: you are deriving `Hash` but have implemented `PartialEq` explicitly" -impl Hash for SqlIdent -where - T: Hash, -{ +// This Hash implementation (and the following) one is required in order to be consistent with PartialEq. +impl Hash for SqlIdent<&Ident> { + fn hash(&self, state: &mut H) { + match self.0.quote_style { + Some(ch) => { + state.write_u8(1); + state.write_u32(ch as u32); + state.write(self.0.value.as_bytes()); + } + None => { + state.write_u8(0); + for ch in self.0.value.chars().flat_map(|ch| ch.to_lowercase()) { + state.write_u32(ch as u32); + } + } + } + } +} + +impl Hash for SqlIdent { fn hash(&self, state: &mut H) { - self.0.hash(state) + SqlIdent(&self.0).hash(state) } } diff --git a/packages/eql-mapper/src/model/type_system.rs b/packages/eql-mapper/src/model/type_system.rs index 7ea05ce9..a4ef7fe4 100644 --- a/packages/eql-mapper/src/model/type_system.rs +++ b/packages/eql-mapper/src/model/type_system.rs @@ -61,6 +61,13 @@ impl Projection { Projection::WithColumns(columns) } } + + pub fn type_at_col_index(&self, index: usize) -> Option<&Value> { + match self { + Projection::WithColumns(cols) => cols.get(index).map(|col| &col.ty), + Projection::Empty => None, + } + } } /// A column from a projection which has a type and an optional alias. diff --git a/packages/eql-mapper/src/test_helpers.rs b/packages/eql-mapper/src/test_helpers.rs index cf36ba46..e183f4fd 100644 --- a/packages/eql-mapper/src/test_helpers.rs +++ b/packages/eql-mapper/src/test_helpers.rs @@ -1,9 +1,12 @@ -use std::fmt::Debug; +use std::{collections::HashMap, convert::Infallible, fmt::Debug, ops::ControlFlow}; -use sqltk::parser::{ - ast::{self as ast, Statement}, - dialect::PostgreSqlDialect, - parser::Parser, +use sqltk::{ + parser::{ + ast::{self as ast, Statement, Value}, + dialect::PostgreSqlDialect, + parser::Parser, + }, + AsNodeKey, Break, NodeKey, Visitable, Visitor, }; use tracing_subscriber::fmt::format; use tracing_subscriber::fmt::format::FmtSpan; @@ -27,7 +30,7 @@ pub(crate) fn init_tracing() { }); } -pub(crate) fn parse(statement: &'static str) -> Statement { +pub(crate) fn parse(statement: &str) -> Statement { Parser::parse_sql(&PostgreSqlDialect {}, statement).unwrap()[0].clone() } @@ -35,6 +38,62 @@ pub(crate) fn id(ident: &str) -> ast::Ident { ast::Ident::from(ident) } +pub(crate) fn get_node_key_of_json_selector<'ast>( + statement: &'ast Statement, + selector: &Value, +) -> NodeKey<'ast> { + find_nodekey_for_value_node(statement, selector.clone()) + .expect("could not find selector Value node") +} + +pub(crate) fn dummy_encrypted_json_selector( + statement: &Statement, + selector: Value, +) -> HashMap, ast::Value> { + if let Value::SingleQuotedString(s) = &selector { + HashMap::from_iter(vec![( + get_node_key_of_json_selector(statement, &selector), + ast::Value::SingleQuotedString(format!("", s)), + )]) + } else { + panic!("dummy_encrypted_json_selector only works on Value::SingleQuotedString") + } +} + +/// Utility for finding the [`NodeKey`] of a [`Value`] node in `statement` by providing a `matching` equal node to search for. +pub(crate) fn find_nodekey_for_value_node( + statement: &Statement, + matching: ast::Value, +) -> Option> { + struct FindNode<'ast> { + needle: ast::Value, + found: Option>, + } + + impl<'a> Visitor<'a> for FindNode<'a> { + type Error = Infallible; + + fn enter(&mut self, node: &'a N) -> ControlFlow> { + if let Some(haystack) = node.downcast_ref::() { + if haystack == &self.needle { + self.found = Some(haystack.as_node_key()); + return ControlFlow::Break(Break::Finished); + } + } + ControlFlow::Continue(()) + } + } + + let mut visitor = FindNode { + needle: matching, + found: None, + }; + + let _ = statement.accept(&mut visitor); + + visitor.found +} + #[macro_export] macro_rules! col { ((NATIVE)) => { diff --git a/packages/eql-mapper/src/transformation_rules/replace_plaintext_eql_literals.rs b/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs similarity index 74% rename from packages/eql-mapper/src/transformation_rules/replace_plaintext_eql_literals.rs rename to packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs index 687649af..af195130 100644 --- a/packages/eql-mapper/src/transformation_rules/replace_plaintext_eql_literals.rs +++ b/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs @@ -1,34 +1,35 @@ use std::{any::type_name, collections::HashMap}; -use sqltk::parser::ast::Value; +use sqltk::parser::ast::{Expr, Value}; use sqltk::{NodeKey, NodePath, Visitable}; use crate::EqlMapperError; +use super::helpers::cast_as_encrypted; use super::TransformationRule; #[derive(Debug)] -pub struct ReplacePlaintextEqlLiterals<'ast> { +pub struct CastLiteralsAsEncrypted<'ast> { encrypted_literals: HashMap, Value>, } -impl<'ast> ReplacePlaintextEqlLiterals<'ast> { +impl<'ast> CastLiteralsAsEncrypted<'ast> { pub fn new(encrypted_literals: HashMap, Value>) -> Self { Self { encrypted_literals } } } -impl<'ast> TransformationRule<'ast> for ReplacePlaintextEqlLiterals<'ast> { +impl<'ast> TransformationRule<'ast> for CastLiteralsAsEncrypted<'ast> { fn apply( &mut self, node_path: &NodePath<'ast>, target_node: &mut N, ) -> Result { if self.would_edit(node_path, target_node) { - if let Some((value,)) = node_path.last_1_as::() { + if let Some((Expr::Value(value),)) = node_path.last_1_as::() { if let Some(replacement) = self.encrypted_literals.remove(&NodeKey::new(value)) { - let target_node = target_node.downcast_mut::().unwrap(); - *target_node = replacement; + let target_node = target_node.downcast_mut::().unwrap(); + *target_node = cast_as_encrypted(replacement); return Ok(true); } } @@ -38,7 +39,7 @@ impl<'ast> TransformationRule<'ast> for ReplacePlaintextEqlLiterals<'ast> { } fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { - if let Some((value,)) = node_path.last_1_as::() { + if let Some((Expr::Value(value),)) = node_path.last_1_as::() { return self.encrypted_literals.contains_key(&NodeKey::new(value)); } false diff --git a/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs b/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs new file mode 100644 index 00000000..ee89fa6e --- /dev/null +++ b/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs @@ -0,0 +1,51 @@ +use super::helpers::cast_as_encrypted; +use super::TransformationRule; +use crate::{EqlMapperError, Type}; +use sqltk::parser::ast::{Expr, Value}; +use sqltk::{NodeKey, NodePath, Visitable}; +use std::collections::HashMap; +use std::sync::Arc; + +#[derive(Debug)] +pub struct CastParamsAsEncrypted<'ast> { + node_types: Arc, Type>>, +} + +impl<'ast> CastParamsAsEncrypted<'ast> { + pub fn new(node_types: Arc, Type>>) -> Self { + Self { node_types } + } +} + +impl<'ast> TransformationRule<'ast> for CastParamsAsEncrypted<'ast> { + fn apply( + &mut self, + node_path: &NodePath<'ast>, + target_node: &mut N, + ) -> Result { + if self.would_edit(node_path, target_node) { + if let Some(expr @ Expr::Value(Value::Placeholder(_))) = target_node.downcast_mut() { + let to_wrap = std::mem::replace(expr, Expr::Value(Value::Null)); + let Expr::Value(value @ Value::Placeholder(_)) = to_wrap else { + unreachable!("the Expr is known to be Expr::Value(Value::Placeholder(_))") + }; + + *expr = cast_as_encrypted(value); + return Ok(true); + } + } + + Ok(false) + } + + fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { + if let Some((node @ Expr::Value(Value::Placeholder(_)),)) = node_path.last_1_as() { + if let Some(Type::Value(crate::Value::Eql(_))) = + self.node_types.get(&NodeKey::new(node)) + { + return true; + } + } + false + } +} diff --git a/packages/eql-mapper/src/transformation_rules/group_by_eql_col.rs b/packages/eql-mapper/src/transformation_rules/group_by_eql_col.rs index 10840a44..78c2f1d6 100644 --- a/packages/eql-mapper/src/transformation_rules/group_by_eql_col.rs +++ b/packages/eql-mapper/src/transformation_rules/group_by_eql_col.rs @@ -40,7 +40,7 @@ impl<'ast> TransformationRule<'ast> for GroupByEqlCol<'ast> { *target_node = helpers::wrap_in_1_arg_function( transformed_expr, - ObjectName(vec![Ident::new("CS_ORE_64_8_V1")]), + ObjectName(vec![Ident::new("eql_v2"), Ident::new("ore_64_8_v2")]), ); return Ok(true); diff --git a/packages/eql-mapper/src/transformation_rules/helpers.rs b/packages/eql-mapper/src/transformation_rules/helpers.rs index d665db57..c070ed8d 100644 --- a/packages/eql-mapper/src/transformation_rules/helpers.rs +++ b/packages/eql-mapper/src/transformation_rules/helpers.rs @@ -1,8 +1,8 @@ use std::{collections::HashMap, convert::Infallible, ops::ControlFlow}; use sqltk::parser::ast::{ - Expr, Function, FunctionArg, FunctionArgExpr, FunctionArgumentList, FunctionArguments, - GroupByExpr, ObjectName, + CastKind, DataType, Expr, Function, FunctionArg, FunctionArgExpr, FunctionArgumentList, + FunctionArguments, GroupByExpr, Ident, ObjectName, }; use sqltk::{AsNodeKey, Break, NodeKey, Visitable, Visitor}; @@ -23,7 +23,7 @@ pub(crate) fn is_used_in_group_by_clause<'ast, N: AsNodeKey>( ty: needle, found: false, }; - exprs.accept(&mut visitor); + let _ = exprs.accept(&mut visitor); visitor.found } }, @@ -48,6 +48,24 @@ pub(crate) fn wrap_in_1_arg_function(expr: Expr, name: ObjectName) -> Expr { }) } +pub(crate) fn cast_as_encrypted(wrapped: sqltk::parser::ast::Value) -> Expr { + let cast_jsonb = Expr::Cast { + kind: CastKind::DoubleColon, + expr: Box::new(Expr::Value(wrapped)), + data_type: DataType::JSONB, + format: None, + }; + + let encrypted_type = ObjectName(vec![Ident::new("eql_v2_encrypted")]); + + Expr::Cast { + kind: CastKind::DoubleColon, + expr: Box::new(cast_jsonb), + data_type: DataType::Custom(encrypted_type, vec![]), + format: None, + } +} + struct ContainsExprWithType<'ast, 't> { node_types: &'t HashMap, Type>, ty: &'t Type, diff --git a/packages/eql-mapper/src/transformation_rules/mod.rs b/packages/eql-mapper/src/transformation_rules/mod.rs index b735a4fa..33964e9d 100644 --- a/packages/eql-mapper/src/transformation_rules/mod.rs +++ b/packages/eql-mapper/src/transformation_rules/mod.rs @@ -11,21 +11,23 @@ mod helpers; +mod cast_literals_as_encrypted; +mod cast_params_as_encrypted; mod fail_on_placeholder_change; mod group_by_eql_col; mod preserve_effective_aliases; -mod replace_plaintext_eql_literals; -mod use_equivalent_eql_fns_on_eql_types; +mod rewrite_standard_sql_fns_on_eql_types; mod wrap_eql_cols_in_order_by_with_ore_fn; mod wrap_grouped_eql_col_in_aggregate_fn; use std::marker::PhantomData; +pub(crate) use cast_literals_as_encrypted::*; +pub(crate) use cast_params_as_encrypted::*; pub(crate) use fail_on_placeholder_change::*; pub(crate) use group_by_eql_col::*; pub(crate) use preserve_effective_aliases::*; -pub(crate) use replace_plaintext_eql_literals::*; -pub(crate) use use_equivalent_eql_fns_on_eql_types::*; +pub(crate) use rewrite_standard_sql_fns_on_eql_types::*; pub(crate) use wrap_eql_cols_in_order_by_with_ore_fn::*; pub(crate) use wrap_grouped_eql_col_in_aggregate_fn::*; diff --git a/packages/eql-mapper/src/transformation_rules/preserve_effective_aliases.rs b/packages/eql-mapper/src/transformation_rules/preserve_effective_aliases.rs index c26445cc..72c29cb1 100644 --- a/packages/eql-mapper/src/transformation_rules/preserve_effective_aliases.rs +++ b/packages/eql-mapper/src/transformation_rules/preserve_effective_aliases.rs @@ -75,18 +75,16 @@ impl PreserveEffectiveAliases { fn effective_aliases_differ(source_node: &SelectItem, target_node: &SelectItem) -> bool { let effective_source_alias = Self::derive_effective_alias(source_node); let effective_target_alias = Self::derive_effective_alias(target_node); - match target_node { - // The captured binding `expr` has type `&mut Expr` but we need an owned `Expr`. to avoid cloning `expr` - // (which can be arbitrarily large) we replace it with another which in return provides us with ownership of - // the original value. `Expr::Wildcard` is chosen as the throwaway value because it's cheap. - SelectItem::UnnamedExpr(_) => { - if let (Some(effective_target_alias), Some(effective_source_alias)) = - (effective_target_alias, effective_source_alias) - { - return effective_target_alias != effective_source_alias; - } + + // The captured binding `expr` has type `&mut Expr` but we need an owned `Expr`. to avoid cloning `expr` + // (which can be arbitrarily large) we replace it with another which in return provides us with ownership of + // the original value. `Expr::Wildcard` is chosen as the throwaway value because it's cheap. + if let SelectItem::UnnamedExpr(_) = target_node { + if let (Some(effective_target_alias), Some(effective_source_alias)) = + (effective_target_alias, effective_source_alias) + { + return effective_target_alias != effective_source_alias; } - _ => {} } false @@ -98,31 +96,29 @@ impl PreserveEffectiveAliases { ) -> bool { let effective_source_alias = Self::derive_effective_alias(source_node); let effective_target_alias = Self::derive_effective_alias(target_node); - match target_node { - // The captured binding `expr` has type `&mut Expr` but we need an owned `Expr`. to avoid cloning `expr` - // (which can be arbitrarily large) we replace it with another which in return provides us with ownership of - // the original value. `Expr::Wildcard` is chosen as the throwaway value because it's cheap. - SelectItem::UnnamedExpr(expr) => { - if let (Some(effective_target_alias), Some(effective_source_alias)) = - (effective_target_alias, effective_source_alias) - { - if effective_target_alias != effective_source_alias { - *target_node = SelectItem::ExprWithAlias { - expr: mem::replace( - expr, - Expr::Wildcard(AttachedToken(TokenWithSpan::new( - Token::EOF, - Span::empty(), - ))), - ), - alias: effective_source_alias, - }; - - return true; - } + + // The captured binding `expr` has type `&mut Expr` but we need an owned `Expr`. to avoid cloning `expr` + // (which can be arbitrarily large) we replace it with another which in return provides us with ownership of + // the original value. `Expr::Wildcard` is chosen as the throwaway value because it's cheap. + if let SelectItem::UnnamedExpr(expr) = target_node { + if let (Some(effective_target_alias), Some(effective_source_alias)) = + (effective_target_alias, effective_source_alias) + { + if effective_target_alias != effective_source_alias { + *target_node = SelectItem::ExprWithAlias { + expr: mem::replace( + expr, + Expr::Wildcard(AttachedToken(TokenWithSpan::new( + Token::EOF, + Span::empty(), + ))), + ), + alias: effective_source_alias, + }; + + return true; } } - _ => {} } false diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_standard_sql_fns_on_eql_types.rs b/packages/eql-mapper/src/transformation_rules/rewrite_standard_sql_fns_on_eql_types.rs new file mode 100644 index 00000000..8dd12c09 --- /dev/null +++ b/packages/eql-mapper/src/transformation_rules/rewrite_standard_sql_fns_on_eql_types.rs @@ -0,0 +1,75 @@ +use std::mem; +use std::{collections::HashMap, sync::Arc}; + +use sqltk::parser::ast::{Expr, Function, Ident, ObjectName}; +use sqltk::{AsNodeKey, NodeKey, NodePath, Visitable}; + +use crate::{ + get_sql_function_def, CompoundIdent, EqlMapperError, RewriteRule, SqlFunction, Type, Value, +}; + +use super::TransformationRule; + +#[derive(Debug)] +pub struct RewriteStandardSqlFnsOnEqlTypes<'ast> { + node_types: Arc, Type>>, +} + +impl<'ast> RewriteStandardSqlFnsOnEqlTypes<'ast> { + pub fn new(node_types: Arc, Type>>) -> Self { + Self { node_types } + } +} + +impl<'ast> TransformationRule<'ast> for RewriteStandardSqlFnsOnEqlTypes<'ast> { + fn apply( + &mut self, + node_path: &NodePath<'ast>, + target_node: &mut N, + ) -> Result { + if self.would_edit(node_path, target_node) { + if let Some((_expr, function)) = node_path.last_2_as::() { + if matches!( + self.node_types.get(&function.as_node_key()), + Some(Type::Value(Value::Eql(_))) + ) { + let function_name = CompoundIdent::from(&function.name.0); + + if let Some(SqlFunction { + rewrite_rule: RewriteRule::AsEqlFunction, + .. + }) = get_sql_function_def(&function_name, &function.args) + { + let function = target_node.downcast_mut::().unwrap(); + let mut existing_name = mem::take(&mut function.name.0); + existing_name.insert(0, Ident::new("eql_v2")); + function.name = ObjectName(existing_name); + } + } + } + } + + Ok(false) + } + + fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { + if let Some((_expr, function)) = node_path.last_2_as::() { + if matches!( + self.node_types.get(&function.as_node_key()), + Some(Type::Value(Value::Eql(_))) + ) { + let function_name = CompoundIdent::from(&function.name.0); + + if let Some(SqlFunction { + rewrite_rule: RewriteRule::AsEqlFunction, + .. + }) = get_sql_function_def(&function_name, &function.args) + { + return true; + } + } + } + + false + } +} diff --git a/packages/eql-mapper/src/transformation_rules/use_equivalent_eql_fns_on_eql_types.rs b/packages/eql-mapper/src/transformation_rules/use_equivalent_eql_fns_on_eql_types.rs deleted file mode 100644 index 6de21b74..00000000 --- a/packages/eql-mapper/src/transformation_rules/use_equivalent_eql_fns_on_eql_types.rs +++ /dev/null @@ -1,73 +0,0 @@ -use std::{collections::HashMap, sync::Arc}; - -use sqltk::parser::ast::{Expr, Function, Ident, Select, SelectItem}; -use sqltk::{NodeKey, NodePath, Visitable}; - -use crate::{EqlMapperError, SqlIdent, Type}; - -use super::{helpers, TransformationRule}; - -#[derive(Debug)] -pub struct UseEquivalentSqlFuncForEqlTypes<'ast> { - node_types: Arc, Type>>, -} - -impl<'ast> UseEquivalentSqlFuncForEqlTypes<'ast> { - pub fn new(node_types: Arc, Type>>) -> Self { - Self { node_types } - } -} - -impl<'ast> TransformationRule<'ast> for UseEquivalentSqlFuncForEqlTypes<'ast> { - fn apply( - &mut self, - node_path: &NodePath<'ast>, - target_node: &mut N, - ) -> Result { - if self.would_edit(node_path, target_node) { - if let Some((_select, _select_items, _select_item, _expr)) = - node_path.last_4_as::, SelectItem, Expr>() - { - let target_node = target_node.downcast_mut::().unwrap(); - if let Expr::Function(Function { name, .. }) = target_node { - let f_name = name.0.last_mut().unwrap(); - - if SqlIdent(&*f_name) == SqlIdent(Ident::new("MIN")) { - *f_name = Ident::new("CS_MIN_V1"); - } - - if SqlIdent(&*f_name) == SqlIdent(Ident::new("MAX")) { - *f_name = Ident::new("CS_MAX_V1"); - } - - return Ok(true); - } - } - } - - Ok(false) - } - - fn would_edit(&mut self, node_path: &NodePath<'ast>, target_node: &N) -> bool { - if let Some((select, _select_items, _select_item, expr)) = - node_path.last_4_as::, SelectItem, Expr>() - { - if !helpers::is_used_in_group_by_clause(&self.node_types, &select.group_by, expr) { - let target_node = target_node.downcast_ref::().unwrap(); - if let Expr::Function(Function { name, .. }) = target_node { - let f_name = name.0.last().unwrap(); - - if SqlIdent(f_name) == SqlIdent(Ident::new("MIN")) { - return true; - } - - if SqlIdent(f_name) == SqlIdent(Ident::new("MAX")) { - return true; - } - } - } - } - - false - } -} diff --git a/packages/eql-mapper/src/transformation_rules/wrap_eql_cols_in_order_by_with_ore_fn.rs b/packages/eql-mapper/src/transformation_rules/wrap_eql_cols_in_order_by_with_ore_fn.rs index 17078487..f899a84f 100644 --- a/packages/eql-mapper/src/transformation_rules/wrap_eql_cols_in_order_by_with_ore_fn.rs +++ b/packages/eql-mapper/src/transformation_rules/wrap_eql_cols_in_order_by_with_ore_fn.rs @@ -11,7 +11,7 @@ use crate::{EqlMapperError, Type, Value}; use super::{helpers::wrap_in_1_arg_function, TransformationRule}; /// When an [`Expr`] of a [`SelectItem`] has an EQL type and that EQL type is used in a `GROUP BY` clause then -/// this rule wraps the `Expr` in a call to `CS_GROUPED_VALUE_V1`. +/// this rule wraps the `Expr` in a call to `eql_v2.grouped_value`. /// /// # Example /// @@ -20,11 +20,11 @@ use super::{helpers::wrap_in_1_arg_function, TransformationRule}; /// SELECT eql_col FROM some_table GROUP BY eql_col; /// /// -- after mapping -/// SELECT CS_GROUPED_VALUE_V1(eql_col) FROM some_table GROUP BY CS_ORE_64_8_V1(eql_col); -/// -- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^ -/// -- ^ ^ -/// -- | | -/// -- Changed by this rule Changed by rule `GroupByEqlCol` +/// SELECT eql_v2.grouped_value(eql_col) AS eql_col FROM some_table GROUP BY eql_v2.cs_ore_64_8(eql_col); +/// -- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +/// -- ^ ^ ^ +/// -- | | | +/// -- Changed by this rule Preserve effective aliases Changed by rule GroupByEqlCol /// ``` #[derive(Debug)] pub struct WrapEqlColsInOrderByWithOreFn<'ast> { @@ -54,7 +54,7 @@ impl<'ast> TransformationRule<'ast> for WrapEqlColsInOrderByWithOreFn<'ast> { target_node.expr = wrap_in_1_arg_function( expr_to_wrap, - ObjectName(vec![Ident::new("CS_ORE_64_8_V1")]), + ObjectName(vec![Ident::new("eql_v2"), Ident::new("ore_64_8_v2")]), ); return Ok(true); diff --git a/packages/eql-mapper/src/transformation_rules/wrap_grouped_eql_col_in_aggregate_fn.rs b/packages/eql-mapper/src/transformation_rules/wrap_grouped_eql_col_in_aggregate_fn.rs index 22d4aeb6..82dfa428 100644 --- a/packages/eql-mapper/src/transformation_rules/wrap_grouped_eql_col_in_aggregate_fn.rs +++ b/packages/eql-mapper/src/transformation_rules/wrap_grouped_eql_col_in_aggregate_fn.rs @@ -20,11 +20,11 @@ use super::{ /// SELECT eql_col FROM some_table GROUP BY eql_col; /// /// -- after mapping -/// SELECT CS_GROUPED_VALUE_V1(eql_col) FROM some_table GROUP BY CS_ORE_64_8_V1(eql_col); -/// -- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^ -/// -- ^ ^ -/// -- | | -/// -- Changed by this rule Changed by rule `GroupByEqlCol` +/// SELECT eql_v2.grouped_value(eql_col) AS eql_col FROM some_table GROUP BY eql_v2.ore_64_8_v2(eql_col); +/// -- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^ +/// -- ^ ^ ^ +/// -- | | | +/// -- Changed by this rule PreserveEffectiveAliases GroupByEqlCol /// ``` #[derive(Debug)] pub struct WrapGroupedEqlColInAggregateFn<'ast> { @@ -50,7 +50,7 @@ impl<'ast> TransformationRule<'ast> for WrapGroupedEqlColInAggregateFn<'ast> { let target_node: &mut Expr = target_node.downcast_mut().unwrap(); *target_node = wrap_in_1_arg_function( expr.clone(), - ObjectName(vec![Ident::new("CS_GROUPED_VALUE_V1")]), + ObjectName(vec![Ident::new("eql_v2"), Ident::new("grouped_value")]), ); return Ok(true); diff --git a/packages/eql-mapper/src/type_checked_statement.rs b/packages/eql-mapper/src/type_checked_statement.rs index bb66aef8..425390cf 100644 --- a/packages/eql-mapper/src/type_checked_statement.rs +++ b/packages/eql-mapper/src/type_checked_statement.rs @@ -4,10 +4,10 @@ use sqltk::parser::ast::{self, Statement}; use sqltk::{AsNodeKey, NodeKey, Transformable}; use crate::{ - DryRunnable, EqlMapperError, EqlValue, FailOnPlaceholderChange, GroupByEqlCol, Param, - PreserveEffectiveAliases, Projection, ReplacePlaintextEqlLiterals, TransformationRule, Type, - UseEquivalentSqlFuncForEqlTypes, Value, WrapEqlColsInOrderByWithOreFn, - WrapGroupedEqlColInAggregateFn, + CastLiteralsAsEncrypted, CastParamsAsEncrypted, DryRunnable, EqlMapperError, EqlValue, + FailOnPlaceholderChange, GroupByEqlCol, Param, PreserveEffectiveAliases, Projection, + RewriteStandardSqlFnsOnEqlTypes, TransformationRule, Type, Value, + WrapEqlColsInOrderByWithOreFn, WrapGroupedEqlColInAggregateFn, }; /// A `TypeCheckedStatement` is returned from a successful call to [`crate::type_check`]. @@ -113,11 +113,7 @@ impl<'ast> TypeCheckedStatement<'ast> { } for (key, _) in encrypted_literals.iter() { - if !self - .literals - .iter() - .any(|(_, node)| &node.as_node_key() == key) - { + if !self.literal_exists_for_node_key(*key) { return Err(EqlMapperError::Transform(String::from( "encrypted literals refers to a literal node which is not present in the SQL statement" ))); @@ -126,6 +122,12 @@ impl<'ast> TypeCheckedStatement<'ast> { Ok(()) } + fn literal_exists_for_node_key(&self, key: NodeKey<'ast>) -> bool { + self.literals + .iter() + .any(|(_, node)| node.as_node_key() == key) + } + fn count_not_null_literals(&self) -> usize { self.literals .iter() @@ -138,13 +140,14 @@ impl<'ast> TypeCheckedStatement<'ast> { encrypted_literals: HashMap, sqltk::parser::ast::Value>, ) -> DryRunnable> { DryRunnable::new(( + RewriteStandardSqlFnsOnEqlTypes::new(Arc::clone(&self.node_types)), WrapGroupedEqlColInAggregateFn::new(Arc::clone(&self.node_types)), GroupByEqlCol::new(Arc::clone(&self.node_types)), WrapEqlColsInOrderByWithOreFn::new(Arc::clone(&self.node_types)), PreserveEffectiveAliases, - ReplacePlaintextEqlLiterals::new(encrypted_literals), - UseEquivalentSqlFuncForEqlTypes::new(Arc::clone(&self.node_types)), + CastLiteralsAsEncrypted::new(encrypted_literals), FailOnPlaceholderChange::new(), + CastParamsAsEncrypted::new(Arc::clone(&self.node_types)), )) } } diff --git a/proxy.Dockerfile b/proxy.Dockerfile index 27a9d431..02cea0e5 100644 --- a/proxy.Dockerfile +++ b/proxy.Dockerfile @@ -10,7 +10,7 @@ COPY cipherstash-proxy /usr/local/bin/cipherstash-proxy COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh # Copy EQL install scripts -COPY cipherstash-eql.sql /opt/cipherstash-eql.sql +COPY cipherstash-encrypt.sql /opt/cipherstash-eql.sql # Copy example schema COPY docs/getting-started/schema-example.sql /opt/schema-example.sql diff --git a/tests/benchmark/sql/benchmark-schema.sql b/tests/benchmark/sql/benchmark-schema.sql index fecdf7bf..7a5be642 100644 --- a/tests/benchmark/sql/benchmark-schema.sql +++ b/tests/benchmark/sql/benchmark-schema.sql @@ -1,4 +1,4 @@ -TRUNCATE TABLE cs_configuration_v1; +TRUNCATE TABLE public.eql_v2_configuration; DROP TABLE IF EXISTS benchmark_plaintext; CREATE TABLE benchmark_plaintext ( @@ -11,13 +11,14 @@ DROP TABLE IF EXISTS benchmark_encrypted; CREATE TABLE benchmark_encrypted ( id serial primary key, username text, - email cs_encrypted_v1 + email eql_v2_encrypted ); -SELECT cs_add_column_v1( +SELECT eql_v2.add_column( 'benchmark_encrypted', 'email' ); -SELECT cs_encrypt_v1(); -SELECT cs_activate_v1(); +SELECT eql_v2.encrypt(); +SELECT eql_v2.activate(); + diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 2f0b1111..8b791d68 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -64,6 +64,7 @@ services: - CS_CLIENT_ID=${CS_CLIENT_ID} - CS_PROMETHEUS__ENABLED=${CS_PROMETHEUS__ENABLED:-true} - CS_SERVER__WORKER_THREADS=${CS_SERVER__WORKER_THREADS:-4} + - CS_REGION=${CS_REGION} networks: - postgres deploy: @@ -99,6 +100,7 @@ services: - CS_TLS__PRIVATE_KEY_PATH=${CS_TLS__PRIVATE_KEY:-/etc/cipherstash-proxy/server.key} - CS_SERVER__REQUIRE_TLS=true - CS_PROMETHEUS__ENABLED=${CS_PROMETHEUS__ENABLED:-true} + - CS_REGION=${CS_REGION} volumes: - ./tls/server.cert:/etc/cipherstash-proxy/server.cert diff --git a/tests/python/tests/test_error_messages.py b/tests/python/tests/test_error_messages.py index f16fd07b..af351e6f 100644 --- a/tests/python/tests/test_error_messages.py +++ b/tests/python/tests/test_error_messages.py @@ -54,10 +54,7 @@ def test_encrypted_column_with_no_configuration(): sql = "INSERT INTO unconfigured (id, encrypted_unconfigured) VALUES (%s, %s)" - # This is EQL catching the error and returning it. Details are in docs/errors.md - # When mapping errors are enabled, (enable_mapping_errors or CS_DEVELOPMENT__ENABLE_MAPPING_ERRORS) - # Proxy will return an error that says "Column X in table Y has no Encrypt configuration" - with pytest.raises(psycopg.Error, match=r"Encrypted column missing \w+ \(\w+\) field"): + with pytest.raises(psycopg.Error, match=r"Column 'encrypted_unconfigured' in table 'unconfigured' has no Encrypt configuration."): cursor.execute(sql, [id, val]) diff --git a/tests/sql/schema-uninstall.sql b/tests/sql/schema-uninstall.sql index 3c34ba76..0c5fbeac 100644 --- a/tests/sql/schema-uninstall.sql +++ b/tests/sql/schema-uninstall.sql @@ -1,4 +1,4 @@ -DROP TABLE IF EXISTS cs_configuration_v1; +DROP TABLE IF EXISTS public.eql_v2_configuration; -- Regular old table DROP TABLE IF EXISTS plaintext; diff --git a/tests/sql/schema.sql b/tests/sql/schema.sql index c1398811..b23cbf74 100644 --- a/tests/sql/schema.sql +++ b/tests/sql/schema.sql @@ -1,4 +1,4 @@ -TRUNCATE TABLE cs_configuration_v1; +TRUNCATE TABLE public.eql_v2_configuration; -- Regular old table DROP TABLE IF EXISTS plaintext; @@ -13,95 +13,95 @@ DROP TABLE IF EXISTS encrypted; CREATE TABLE encrypted ( id bigint, plaintext text, - encrypted_text cs_encrypted_v1, - encrypted_bool cs_encrypted_v1, - encrypted_int2 cs_encrypted_v1, - encrypted_int4 cs_encrypted_v1, - encrypted_int8 cs_encrypted_v1, - encrypted_float8 cs_encrypted_v1, - encrypted_date cs_encrypted_v1, - encrypted_jsonb cs_encrypted_v1, + encrypted_text eql_v2_encrypted, + encrypted_bool eql_v2_encrypted, + encrypted_int2 eql_v2_encrypted, + encrypted_int4 eql_v2_encrypted, + encrypted_int8 eql_v2_encrypted, + encrypted_float8 eql_v2_encrypted, + encrypted_date eql_v2_encrypted, + encrypted_jsonb eql_v2_encrypted, PRIMARY KEY(id) ); DROP TABLE IF EXISTS unconfigured; CREATE TABLE unconfigured ( id bigint, - encrypted_unconfigured cs_encrypted_v1, + encrypted_unconfigured eql_v2_encrypted, PRIMARY KEY(id) ); -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'encrypted', 'encrypted_text', 'unique', 'text' ); -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'encrypted', 'encrypted_text', 'match', 'text' ); -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'encrypted', 'encrypted_text', 'ore', 'text' ); -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'encrypted', 'encrypted_bool', 'unique', 'boolean' ); -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'encrypted', 'encrypted_bool', 'ore', 'boolean' ); -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'encrypted', 'encrypted_int2', 'unique', 'small_int' ); -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'encrypted', 'encrypted_int2', 'ore', 'small_int' ); -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'encrypted', 'encrypted_int4', 'unique', 'int' ); -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'encrypted', 'encrypted_int4', 'ore', 'int' ); -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'encrypted', 'encrypted_int8', 'unique', 'big_int' ); -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'encrypted', 'encrypted_int8', 'ore', @@ -109,35 +109,35 @@ SELECT cs_add_index_v1( ); -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'encrypted', 'encrypted_float8', 'unique', 'double' ); -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'encrypted', 'encrypted_float8', 'ore', 'double' ); -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'encrypted', 'encrypted_date', 'unique', 'date' ); -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'encrypted', 'encrypted_date', 'ore', 'date' ); -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'encrypted', 'encrypted_jsonb', 'ste_vec', @@ -145,5 +145,5 @@ SELECT cs_add_index_v1( '{"prefix": "encrypted/encrypted_jsonb"}' ); -SELECT cs_encrypt_v1(); -SELECT cs_activate_v1(); +SELECT eql_v2.encrypt(); +SELECT eql_v2.activate(); diff --git a/tests/tasks/test/integration/psql-passthrough.sh b/tests/tasks/test/integration/psql-passthrough.sh index e07fd77d..96ec1e4e 100755 --- a/tests/tasks/test/integration/psql-passthrough.sh +++ b/tests/tasks/test/integration/psql-passthrough.sh @@ -17,10 +17,10 @@ EOF # Confirm that there is indeed no config set +e -OUTPUT="$(docker exec -i postgres${CONTAINER_SUFFIX} psql 'postgresql://cipherstash:password@proxy:6432/cipherstash?sslmode=disable' --command 'SELECT * FROM cs_configuration_v1' 2>&1)" +OUTPUT="$(docker exec -i postgres${CONTAINER_SUFFIX} psql 'postgresql://cipherstash:password@proxy:6432/cipherstash?sslmode=disable' --command 'SELECT * FROM eql_v2_configuration' 2>&1)" retval=$? -if echo ${OUTPUT} | grep -v 'relation "cs_configuration_v1" does not exist'; then - echo "error: did not see string in output: \"relation "cs_configuration_v1" does not exist\"" +if echo ${OUTPUT} | grep -v 'relation "eql_v2_configuration" does not exist'; then + echo "error: did not see string in output: \"relation "eql_v2_configuration" does not exist\"" exit 1 fi