Merge pull request #228 from cipherstash/handle-datarow-with-binary-encoding-and-null

Handle data row with binary encoding and null

Author: tobyhede

packages/cipherstash-proxy-integration/src/extended_protocol_error_messages.rs

@@ -70,7 +70,7 @@ mod tests {
             // returns eql_v1_encrypted and the client cannot convert to a string.
             // If mapping errors are enabled (enable_mapping_errors or CS_DEVELOPMENT__ENABLE_MAPPING_ERRORS),
             // then Proxy will return an error that says "Column X in table Y has no Encrypt configuration"
-            assert_eq!(msg, "error serializing parameter 1: cannot convert between the Rust type `&str` and the Postgres type `eql_v1_encrypted`");
+            assert_eq!(msg, "error serializing parameter 1: cannot convert between the Rust type `&str` and the Postgres type `jsonb`");
         } else {
             unreachable!();
         }

packages/cipherstash-proxy/src/postgresql/backend.rs

@@ -246,7 +246,7 @@ where
             }
         };
 
-        let rows: Vec<DataRow> = self.buffer.drain().into_iter().collect();
+        let mut rows: Vec<DataRow> = self.buffer.drain().into_iter().collect();
         debug!(target: DEVELOPMENT, client_id = self.context.client_id, rows = rows.len());
 
         let result_column_count = match rows.first() {
@@ -264,7 +264,7 @@ where
 
         // Each row is converted into Vec<Option<CipherText>>
         let ciphertexts: Vec<Option<EqlEncrypted>> = rows
-            .iter()
+            .iter_mut()
             .flat_map(|row| row.to_ciphertext(projection_columns))
             .collect::<Vec<_>>();
 

packages/cipherstash-proxy/src/postgresql/messages/data_row.rs

@@ -21,11 +21,11 @@ pub struct DataColumn {
 
 impl DataRow {
     pub fn to_ciphertext(
-        &self,
+        &mut self,
         column_configuration: &Vec<Option<Column>>,
     ) -> Vec<Option<eql::EqlEncrypted>> {
         let mut result = vec![];
-        for (data_column, column_config) in self.columns.iter().zip(column_configuration) {
+        for (data_column, column_config) in self.columns.iter_mut().zip(column_configuration) {
             let encrypted = column_config
                 .as_ref()
                 .filter(|_| data_column.is_not_null())
@@ -35,6 +35,7 @@ impl DataRow {
                         .inspect_err(|err| match err {
                             Error::Encrypt(EncryptError::ColumnIsNull) => {
                                 // Not an error, as you were
+                                data_column.set_null();
                             }
                             _ => {
                                 let err = EncryptError::ColumnCouldNotBeDeserialised {
@@ -80,6 +81,10 @@ impl DataColumn {
         self.bytes.is_some()
     }
 
+    pub fn set_null(&mut self) {
+        self.bytes = None;
+    }
+
     pub fn rewrite(&mut self, b: &[u8]) {
         if let Some(ref mut bytes) = self.bytes {
             bytes.clear();
@@ -169,10 +174,10 @@ impl TryFrom<DataColumn> for BytesMut {
     }
 }
 
-impl TryFrom<&DataColumn> for eql::EqlEncrypted {
+impl TryFrom<&mut DataColumn> for eql::EqlEncrypted {
     type Error = Error;
 
-    fn try_from(col: &DataColumn) -> Result<Self, Error> {
+    fn try_from(col: &mut DataColumn) -> Result<Self, Error> {
         if let Some(bytes) = &col.bytes {
             if &bytes[0..=1] == b"(\"" {
                 // Text encoding
@@ -230,75 +235,143 @@ impl TryFrom<&DataColumn> for eql::EqlEncrypted {
 
 #[cfg(test)]
 mod tests {
-    use std::io::Cursor;
-
     use super::DataRow;
+    use crate::Identifier;
     use crate::{
         config::{LogConfig, LogLevel},
         log,
-        postgresql::messages::data_row::DataColumn,
+        postgresql::{data, messages::data_row::DataColumn, Column},
     };
-    use crate::{EqlEncrypted, Identifier};
     use bytes::BytesMut;
-    use tracing::info;
+    use cipherstash_client::schema::{ColumnConfig, ColumnType};
 
     fn to_message(s: &[u8]) -> BytesMut {
         BytesMut::from(s)
     }
 
+    fn column_config(column: &str) -> Option<Column> {
+        let identifier = Identifier::new("encrypted", column);
+        let config = ColumnConfig::build("column".to_string()).casts_as(ColumnType::SmallInt);
+        let column = Column::new(identifier, config);
+        Some(column)
+    }
+
+    fn column_config_with_id(column: &str) -> Vec<Option<Column>> {
+        vec![None, column_config(column)]
+    }
+
     #[test]
-    pub fn parse_encrypted_column_with_binary_encoding() {
-        // Binary
+    pub fn to_ciphertext_with_binary_encoding() {
+        log::init(LogConfig::with_level(LogLevel::Debug));
+
+        //  Binary
         // SELECT id, encrypted_text FROM encrypted WHERE id = $1
         let bytes = to_message(b"D\0\0\nR\0\x02\0\0\0\x08w\xaam\xf8Y$\x9dI\0\0\n<\0\0\0\x01\0\0\x0e\xda\0\0\n0\x01{\"b\": null, \"c\": \"mBbLbP2ww9ymEpm_yfj>@=^)JCqtLxcewai)Ilzx#HbC2p3F;dB`XP9af|s-igMjdMWLYPqYWAB#2|%<Q?A|Izg<&Cs4$4MtatzDN{_NWZFbsdA0=?)lF+VYZewzJaCBv4Uvy=7bie\", \"i\": {\"c\": \"encrypted_text\", \"t\": \"encrypted\"}, \"m\": [71, 1624, 929, 1339, 1764, 1380, 1256, 2018, 575, 470, 1792, 1684, 205, 894, 1365, 272, 814, 1333, 1971, 1942, 1335, 404, 1204, 638, 18, 1147, 1098, 1448, 403, 234, 647, 1982, 279, 1606, 826, 113, 652, 1287, 986, 1239, 1988, 358, 1589, 1775, 1997, 633, 369, 1744, 1700, 1149, 1641, 609, 1506, 1915, 630, 1045, 141, 815, 445, 145, 1758, 1772, 1162, 1761, 1619, 1328, 901, 1090, 637, 1529, 1181, 527, 388, 2015, 1317, 1019, 1369, 1340, 176, 936, 1716, 515, 1101, 656, 1737, 1858, 1633, 1849, 512, 1347, 1389, 700, 1336, 1253, 1396, 381, 35, 586, 581, 1877, 1226, 1273, 80, 1499, 433, 1649, 573, 1150, 1572, 1533, 1077], \"o\": [\"faa1f63cb6d36094d1aa50db6c0217eb447a987071119bb127f677b6a7ee0b4fe40eed7cd84e96e8a11bbe3ea14331f3ec4c8f149ce9d2b0253b4676c86557fcec4a5f8ca4e1ee081c66bf0a3cb594c6b5739f77f62fc5e76991869c23a97f01816cde3dfc24b2ca2fbb12b50fde324f18aa51718d681772bf9caf3c059a6748cbcaf4dd1c4fa026e47f4be75ce9046de508041645c0d48cddef735db92b4495a783b2a0f54d6723c959f74aae6fab62202f0d1f2cc2336ced18df80d12b4c6b5e504d2f6e21e12e2cc8ae620b9c714b8becbed3ed8d2b4ece3f0c911eeee4cba805098becdb041966faf06546cb48037153c3285f3d53f750c7cb7b1b6a985de296d0592b0bcb71687a09cd38d53979c5399245a85f9c8c5db68d14a2c2521795b8d670700e1ff324e0f46fe5338f63074adeba5a8a7d81b2693413ed97aa827b5a16ce9fa33ff9c2870465d992e367dfca76d957cbfb1062433825b83941f40b4d47cd65522c8634f4440b058dae20ec940eecaad70f46a81599ebec6c90735a51170f5456685307e9bdb5d9b94665c6b86985dcd95125\", \"b41d89a196a35252a965ce3c330eac369ead56e9f06e2016da4d6971fe0b8d6e677e1018e7a1bd2fa0b2c1faaa12650d678352ecc81f6be879213fe78b8004b87dd7dcadec59df4dcafdb3c9aa55dcb2cc2bcf2193574b201c9a1c14764d69716f63b0c1aa30a2846696f2a1c790ca2cb26370d7e20904a8748ea98a95ee3cbb95c5f342de4e71bb080b6e5cfcb730ba4c094304c759fe520fd59fa20eb1381bf9cb07b3a952a9cd0994ba49085475b605c67df0f5fb970fe20c343bdb6e4e90037656787cfd58622aa6f77026c57b66b95390f7560ed7dc640553e6219dea4015b3484e835241090d1bae888cbf946dc114680e36119a3aa95f64fb13c88dd6e9636898d423d82964dddec5311ec94a30b71eab561988be6cd07e6b7c18df9b4f0fffcc53cec5e883830ab868ee626d7c9bf6bbf8612eb2e0b472e223a06ddf920988630809e02cacf525e7533406f08ec7b192f2d78cb9d4acda1cd8e0d35898ff15b0b6c00d5dc4fa9e45f0666319e32a54d41da63b34cd83b2ccbb70db48ab3d3a959f2e24e40b899e334b8458430376cb7e28c8e673\"], \"s\": null, \"u\": \"962d77dfaf892b596b3255c022359e54f3e8dc8b21c3d1b32ebd05555f433192\", \"v\": 1, \"sv\": null, \"ocf\": null, \"ocv\": null}");
-        let data_row = DataRow::try_from(&bytes).unwrap();
+        let mut data_row = DataRow::try_from(&bytes).unwrap();
+
+        let column_config = column_config_with_id("encrypted_text");
+        let encrypted = data_row.to_ciphertext(&column_config);
+
+        assert_eq!(encrypted.len(), 2);
+
+        // Two rows
+        assert!(encrypted[0].is_none());
+        assert!(encrypted[1].is_some());
 
-        let col = &data_row.columns[1];
-        let e: EqlEncrypted = col.try_into().unwrap();
-        let expected = Identifier::new("encrypted", "encrypted_text");
-        assert_eq!(e.identifier, expected);
+        assert_eq!(
+            column_config[1].as_ref().unwrap().identifier,
+            encrypted[1].as_ref().unwrap().identifier
+        );
     }
 
     #[test]
-    pub fn parse_encrypted_column_with_binary_encoding_and_null() {
+    pub fn to_ciphertext_with_binary_encoding_and_null() {
         log::init(LogConfig::with_level(LogLevel::Debug));
 
         // Binary
         // encrypted_text IS NULL
         // SELECT id, encrypted_text FROM encrypted WHERE id = $1
-        let bytes = to_message(b"D\0\0\0\"\0\x02\0\0\0\x089\"\x88A\xe59\xb0\x13\0\0\0\x0c\0\0\0\x01\0\0\x0e\xda\xff\xff\xff\xff");
-        let data_row = DataRow::try_from(&bytes).unwrap();
 
-        // The error is handled in to_ciphertext and NONE
-        let col = &data_row.columns[1];
-        let result: Result<EqlEncrypted, _> = col.try_into();
+        // let bytes = to_message(b"D\0\0\0\"\0\x02\0\0\0\x089\"\x88A\xe59\xb0\x13\0\0\0\x0c\0\0\0\x01\0\0\x0e\xda\xff\xff\xff\xff");
+        let bytes = to_message(b"D\0\0\0\"\0\x02\0\0\0\x08>\xe6=<Yk\0\r\0\0\0\x0c\0\0\0\x01\0\0\x0e\xda\xff\xff\xff\xff");
+        let mut data_row = DataRow::try_from(&bytes).unwrap();
+
+        assert!(data_row.columns[1].bytes.is_some());
+
+        let column_config = column_config_with_id("encrypted_text");
+        let encrypted = data_row.to_ciphertext(&column_config);
 
-        result.expect_err("Expected parsing of NULL column to fail");
+        assert_eq!(encrypted.len(), 2);
+
+        // Two rows
+        assert!(encrypted[0].is_none());
+        assert!(encrypted[1].is_none());
+
+        // DataColumn has been NULLIFIED
+        assert!(data_row.columns[1].bytes.is_none());
     }
 
     #[test]
-    pub fn parse_encrypted_column_with_text_encoding() {
+    pub fn to_ciphertext_with_text_encoding() {
+        log::init(LogConfig::with_level(LogLevel::Debug));
+
         // SELECT encrypted_jsonb FROM encrypted LIMIT 1
         let bytes = to_message(b"D\0\0\x03\xba\0\x01\0\0\x03\xb0(\"{\"\"b\"\": null, \"\"c\"\": \"\"mBbLR(BvRN1BF^PAFs!B^`U;mA>uOUiFLgDpZXhU#s#%c4wyi&Z7`(d0IxUty-cI#Yp%o~QFF39^sRf>4*EG{zlk;}ArEQ}NQHa9@;T73aPOSTpuh\"\", \"\"i\"\": {\"\"c\"\": \"\"encrypted_jsonb\"\", \"\"t\"\": \"\"encrypted\"\"}, \"\"m\"\": null, \"\"o\"\": null, \"\"s\"\": null, \"\"u\"\": null, \"\"v\"\": 1, \"\"sv\"\": [{\"\"b\"\": \"\"8067db44a848ab32c3056a3dbe4edf16\"\", \"\"c\"\": \"\"mBbLR(BvRN1BF^PAFs!B^`U;mA>uOUiFLgDpZXhU#s#%c4wyi&Z7`(d0IxUty-cI#Yp%o~QFF39^sRf>4*EG{zlk;}ArEQ}NQHa9@;T73aPOSTpuh\"\", \"\"m\"\": null, \"\"o\"\": null, \"\"s\"\": \"\"9493d6010fe7845d52149b697729c745\"\", \"\"u\"\": null, \"\"sv\"\": null, \"\"ocf\"\": null, \"\"ocv\"\": null}, {\"\"b\"\": null, \"\"c\"\": \"\"mBbLR(BvRN1BF^PAFs!B^`U;m8QkTKr|h>Q`^NbW(CC|>SD}UM=o%mz(Fw#LQFF39^sRf>4*EG{zlk;}ArEQ}NQHa9@;T73aPOSTpuh\"\", \"\"m\"\": null, \"\"o\"\": null, \"\"s\"\": \"\"b1f0e4bb3855bc33936ef1fddf532765\"\", \"\"u\"\": null, \"\"sv\"\": null, \"\"ocf\"\": null, \"\"ocv\"\": \"\"fbc7a11fc81f2a31c904c5b05572b054824e3b5f5ece78f1b711f93175f0a4a9726157cea247e107\"\"}], \"\"ocf\"\": null, \"\"ocv\"\": null}\")");
-        let data_row = DataRow::try_from(&bytes).unwrap();
+        let mut data_row = DataRow::try_from(&bytes).unwrap();
 
-        let col = data_row.columns.first().unwrap();
-        let e: EqlEncrypted = col.try_into().unwrap();
-        let expected = Identifier::new("encrypted", "encrypted_jsonb");
-        assert_eq!(e.identifier, expected);
+        assert!(data_row.columns[0].bytes.is_some());
+
+        let column_config = vec![column_config("encrypted_jsonb")];
+        let encrypted = data_row.to_ciphertext(&column_config);
+
+        assert_eq!(encrypted.len(), 1);
+        assert!(encrypted[0].is_some());
+
+        assert_eq!(
+            column_config[0].as_ref().unwrap().identifier,
+            encrypted[0].as_ref().unwrap().identifier
+        );
     }
 
     #[test]
-    pub fn parse_encrypted_column_with_text_encoding_and_null() {
+    pub fn to_ciphertext_with_text_encoding_and_null() {
+        log::init(LogConfig::with_level(LogLevel::Debug));
+
         // SELECT * FROM encrypted WHERE id = $1;
         // Only encrypted_text is NOT NULL
         let bytes = to_message(b"D\0\0\n\x91\0\n\0\0\0\n1297231342\xff\xff\xff\xff\0\0\nY(\"{\"\"b\"\": null, \"\"c\"\": \"\"mBbJ;S^xMu<v?;UyTSS~VfK;4C(U~uOiKbWSK*!hB3vi!C$luW$k`K6>@++(U20{lxK;qYYaDYF#30N~x;wyOUMoFOB9K!>A_9g9j@+M6V3wENqu#H8gDb9OZewzJaCBv4Uvy=7bie\"\", \"\"i\"\": {\"\"c\"\": \"\"encrypted_text\"\", \"\"t\"\": \"\"encrypted\"\"}, \"\"m\"\": [369, 381, 1758, 403, 35, 609, 1181, 1098, 1347, 1633, 1150, 815, 1997, 234, 1858, 656, 1335, 936, 1204, 630, 1764, 1328, 1649, 1396, 113, 1149, 1499, 1147, 586, 1942, 901, 1256, 1226, 1045, 637, 279, 1162, 1077, 1340, 1336, 1448, 700, 176, 1849, 1915, 1389, 71, 515, 633, 388, 1877, 1339, 1239, 638, 1365, 1380, 1273, 581, 1792, 1716, 145, 512, 814, 272, 1333, 1775, 1572, 1744, 2018, 433, 1641, 1529, 647, 1317, 652, 1606, 1737, 470, 826, 80, 929, 1700, 1619, 1253, 358, 1589, 1971, 1019, 1533, 1624, 573, 1684, 1287, 575, 1761, 527, 404, 1369, 894, 18, 1101, 986, 1772, 1090, 1506, 2015, 1988, 205, 141, 445, 1982], \"\"o\"\": [\"\"faa1f63cb6d36094d1aa50db6c0217eb447a987071119bb127f677b6a7ee0b4fe40eed7cd84e96e8a11bbe3ea14331f3ec4c8f149ce9d2b0253b4676c86557fcec4a5f8ca4e1ee081c66bf0a3cb594c6b5739f77f62fc5e76991869c23a97f01816cde3dfc24b2ca2fbb12b50fde324f18aa51718d681772bf9caf3c059a6748cbcaf4dd1c4fa02645d74699d7d265faf938c339f6cc8f57db9bd4cff8e03cae9e5d21a651b33525e86e335dff61520e8f23d7002f05fa186075a335fb7b2c740133b5a72760ccd216127d69983aa31a090a3b6ca56a48b6372cab60c979465d84dc94e5452c92517b643882fa82c22a26b4feaaa1b0ae8fcb989b10d0351fb3c9c5e56e719f820442612a67fff334438f3f5d35ff6db1b5f7a50670c7fec014f6fc19c352eb011911faf62a230e10c2d16f6c84b46cf9ee7eb1afb9c61a523891e31da2a18b445769d75c11873566dc8196d77e985423226bd1db10e4ce9eb10c2f69db7ce57d47281401617978d2bcfca23b9015b9e705615b8bf773daa87a18417f86e5338a7929fa4f10c6864af09870bfd9ddfb7848\"\", \"\"b41d89a196a35252a965ce3c330eac369ead56e9f06e2016da4d6971fe0b8d6e677e1018e7a1bd2fa0b2c1faaa12650d678352ecc81f6be879213fe78b8004b87dd7dcadec59df4dcafdb3c9aa55dcb2cc2bcf2193574b201c9a1c14764d69716f63b0c1aa30a2846696f2a1c790ca2cb26370d7e20904a8748ea98a95ee3cbb95c5f342de4e71bbf0262e84d59188ea72fe4449a16e7c73f88ed06b9cb724902a85d063c03e9b1a63dd18b9604625ca3cb8110d9c8f93e1771525c51b6ee092d554e84d61df5b557994f32191bb2b6801d9727fb707d5287e6c83d6b16763a6e66526baf80765a58d36df744be7872d2750eb28a86a519a21ee710f618c09cb2bd45f21e805ae4e11eb2987d7be31c32164d4f828fc35c389d516d0d6a54e25041985cffcb6124b4d3fa5b0ba91e19d60e3102370e9c1c768df1b427c682304a1dfdea2d3e514db22057f43d8121b8daf7c434831e5b618bbca9f4e198741927bdc168e4703fb1f703957f7b70491e06bec4adee19d29ef5e938695e1d49ef50ceef0a9c3e46bd8fe309e013e5ea0d35c5ebf3dddd97573\"\"], \"\"s\"\": null, \"\"u\"\": \"\"962d77dfaf892b596b3255c022359e54f3e8dc8b21c3d1b32ebd05555f433192\"\", \"\"v\"\": 1, \"\"sv\"\": null, \"\"ocf\"\": null, \"\"ocv\"\": null}\")\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff");
-        let data_row = DataRow::try_from(&bytes).unwrap();
 
-        let col = &data_row.columns[2];
-        let e: EqlEncrypted = col.try_into().unwrap();
-        let expected = Identifier::new("encrypted", "encrypted_text");
-        assert_eq!(e.identifier, expected);
+        let mut data_row = DataRow::try_from(&bytes).unwrap();
+
+        assert!(data_row.columns[0].bytes.is_some());
+
+        let column_config = vec![
+            None,
+            None,
+            column_config("encrypted_text"),
+            column_config("encrypted_bool"),
+            column_config("encrypted_int2"),
+            column_config("encrypted_int4"),
+            column_config("encrypted_int8"),
+            column_config("encrypted_float8"),
+            column_config("encrypted_date"),
+            column_config("encrypted_jsonb"),
+        ];
+
+        let encrypted = data_row.to_ciphertext(&column_config);
+
+        assert_eq!(encrypted.len(), 10);
+
+        assert!(encrypted[0].is_none());
+        assert!(encrypted[1].is_none());
+        assert!(encrypted[2].is_some()); // <-- Some
+        assert!(encrypted[3].is_none());
+        // etc
+
+        assert_eq!(
+            column_config[2].as_ref().unwrap().identifier,
+            encrypted[2].as_ref().unwrap().identifier
+        );
     }
 
     #[test]