feat: handle binary encoding of custom types in DataRow

Author: tobyhede

packages/cipherstash-proxy/src/postgresql/messages/data_row.rs

@@ -169,20 +169,36 @@ impl TryFrom<&DataColumn> for eql::EqlEncrypted {
 
     fn try_from(col: &DataColumn) -> Result<Self, Error> {
         if let Some(bytes) = &col.bytes {
-            // Encrypted record is in the form ("{}")
-            // json data can be extracted by dropping the first and last two bytes to remove (" and ")
-            let start = 2;
-            let end = bytes.len() - 2;
-            let sliced = &bytes[start..end];
-
-            let input = String::from_utf8_lossy(sliced).to_string();
-            let input = input.replace("\"\"", "\"");
-
-            match serde_json::from_str(&input) {
-                Ok(e) => return Ok(e),
-                Err(err) => {
-                    debug!(target: DECRYPT, error = err.to_string());
-                    return Err(err.into());
+            if &bytes[0..=1] == b"(\"" {
+                // Text encoding
+                // Encrypted record is in the form ("{}")
+                // json data can be extracted by dropping the first and last two bytes to remove (" and ")
+                let start = 2;
+                let end = bytes.len() - 2;
+                let sliced = &bytes[start..end];
+
+                let input = String::from_utf8_lossy(sliced).to_string();
+                let input = input.replace("\"\"", "\"");
+
+                match serde_json::from_str(&input) {
+                    Ok(e) => return Ok(e),
+                    Err(err) => {
+                        debug!(target: DECRYPT, error = err.to_string());
+                        return Err(err.into());
+                    }
+                }
+            } else {
+                // 12 bytes for the binary rowtype header
+                // 1 byte for the jsonb header (value of 1)
+                let start = 12 + 1;
+                let sliced = &bytes[start..];
+
+                match serde_json::from_slice(sliced) {
+                    Ok(e) => return Ok(e),
+                    Err(err) => {
+                        debug!(target: DECRYPT, error = err.to_string());
+                        return Err(err.into());
+                    }
                 }
             }
         }
@@ -193,6 +209,8 @@ impl TryFrom<&DataColumn> for eql::EqlEncrypted {
 
 #[cfg(test)]
 mod tests {
+    use std::io::Cursor;
+
     use super::DataRow;
     use crate::{
         config::{LogConfig, LogLevel},
@@ -201,6 +219,7 @@ mod tests {
     };
     use crate::{EqlEncrypted, Identifier};
     use bytes::BytesMut;
+    use tracing::info;
 
     fn to_message(s: &[u8]) -> BytesMut {
         BytesMut::from(s)
@@ -209,28 +228,44 @@ mod tests {
     #[test]
     pub fn parse_encrypted_column() {
         log::init(LogConfig::with_level(LogLevel::Debug));
+        use bytes::Buf;
+
+        let bytes = to_message(b"\0\0\0\x01\0\0\x0e\xda\0\0\n0\x01");
+
+        let mut cursor = Cursor::new(bytes);
+        let a = cursor.get_i32();
+        let b = cursor.get_i32();
+        let c = cursor.get_i32();
+
+        info!("{a}, {b}, {c}");
+
+        // Binary
+        // SELECT id, encrypted_text FROM encrypted WHERE id = $1
+        let bytes = to_message(b"D\0\0\nR\0\x02\0\0\0\x08w\xaam\xf8Y$\x9dI\0\0\n<\0\0\0\x01\0\0\x0e\xda\0\0\n0\x01{\"b\": null, \"c\": \"mBbLbP2ww9ymEpm_yfj>@=^)JCqtLxcewai)Ilzx#HbC2p3F;dB`XP9af|s-igMjdMWLYPqYWAB#2|%<Q?A|Izg<&Cs4$4MtatzDN{_NWZFbsdA0=?)lF+VYZewzJaCBv4Uvy=7bie\", \"i\": {\"c\": \"encrypted_text\", \"t\": \"encrypted\"}, \"m\": [71, 1624, 929, 1339, 1764, 1380, 1256, 2018, 575, 470, 1792, 1684, 205, 894, 1365, 272, 814, 1333, 1971, 1942, 1335, 404, 1204, 638, 18, 1147, 1098, 1448, 403, 234, 647, 1982, 279, 1606, 826, 113, 652, 1287, 986, 1239, 1988, 358, 1589, 1775, 1997, 633, 369, 1744, 1700, 1149, 1641, 609, 1506, 1915, 630, 1045, 141, 815, 445, 145, 1758, 1772, 1162, 1761, 1619, 1328, 901, 1090, 637, 1529, 1181, 527, 388, 2015, 1317, 1019, 1369, 1340, 176, 936, 1716, 515, 1101, 656, 1737, 1858, 1633, 1849, 512, 1347, 1389, 700, 1336, 1253, 1396, 381, 35, 586, 581, 1877, 1226, 1273, 80, 1499, 433, 1649, 573, 1150, 1572, 1533, 1077], \"o\": [\"faa1f63cb6d36094d1aa50db6c0217eb447a987071119bb127f677b6a7ee0b4fe40eed7cd84e96e8a11bbe3ea14331f3ec4c8f149ce9d2b0253b4676c86557fcec4a5f8ca4e1ee081c66bf0a3cb594c6b5739f77f62fc5e76991869c23a97f01816cde3dfc24b2ca2fbb12b50fde324f18aa51718d681772bf9caf3c059a6748cbcaf4dd1c4fa026e47f4be75ce9046de508041645c0d48cddef735db92b4495a783b2a0f54d6723c959f74aae6fab62202f0d1f2cc2336ced18df80d12b4c6b5e504d2f6e21e12e2cc8ae620b9c714b8becbed3ed8d2b4ece3f0c911eeee4cba805098becdb041966faf06546cb48037153c3285f3d53f750c7cb7b1b6a985de296d0592b0bcb71687a09cd38d53979c5399245a85f9c8c5db68d14a2c2521795b8d670700e1ff324e0f46fe5338f63074adeba5a8a7d81b2693413ed97aa827b5a16ce9fa33ff9c2870465d992e367dfca76d957cbfb1062433825b83941f40b4d47cd65522c8634f4440b058dae20ec940eecaad70f46a81599ebec6c90735a51170f5456685307e9bdb5d9b94665c6b86985dcd95125\", \"b41d89a196a35252a965ce3c330eac369ead56e9f06e2016da4d6971fe0b8d6e677e1018e7a1bd2fa0b2c1faaa12650d678352ecc81f6be879213fe78b8004b87dd7dcadec59df4dcafdb3c9aa55dcb2cc2bcf2193574b201c9a1c14764d69716f63b0c1aa30a2846696f2a1c790ca2cb26370d7e20904a8748ea98a95ee3cbb95c5f342de4e71bb080b6e5cfcb730ba4c094304c759fe520fd59fa20eb1381bf9cb07b3a952a9cd0994ba49085475b605c67df0f5fb970fe20c343bdb6e4e90037656787cfd58622aa6f77026c57b66b95390f7560ed7dc640553e6219dea4015b3484e835241090d1bae888cbf946dc114680e36119a3aa95f64fb13c88dd6e9636898d423d82964dddec5311ec94a30b71eab561988be6cd07e6b7c18df9b4f0fffcc53cec5e883830ab868ee626d7c9bf6bbf8612eb2e0b472e223a06ddf920988630809e02cacf525e7533406f08ec7b192f2d78cb9d4acda1cd8e0d35898ff15b0b6c00d5dc4fa9e45f0666319e32a54d41da63b34cd83b2ccbb70db48ab3d3a959f2e24e40b899e334b8458430376cb7e28c8e673\"], \"s\": null, \"u\": \"962d77dfaf892b596b3255c022359e54f3e8dc8b21c3d1b32ebd05555f433192\", \"v\": 1, \"sv\": null, \"ocf\": null, \"ocv\": null}");
+        let data_row = DataRow::try_from(&bytes).unwrap();
+
+        let col = &data_row.columns[1];
+        let e: EqlEncrypted = col.try_into().unwrap();
+        let expected = Identifier::new("encrypted", "encrypted_text");
+        assert_eq!(e.identifier, expected);
 
         // SELECT encrypted_jsonb FROM encrypted LIMIT 1
         let bytes = to_message(b"D\0\0\x03\xba\0\x01\0\0\x03\xb0(\"{\"\"b\"\": null, \"\"c\"\": \"\"mBbLR(BvRN1BF^PAFs!B^`U;mA>uOUiFLgDpZXhU#s#%c4wyi&Z7`(d0IxUty-cI#Yp%o~QFF39^sRf>4*EG{zlk;}ArEQ}NQHa9@;T73aPOSTpuh\"\", \"\"i\"\": {\"\"c\"\": \"\"encrypted_jsonb\"\", \"\"t\"\": \"\"encrypted\"\"}, \"\"m\"\": null, \"\"o\"\": null, \"\"s\"\": null, \"\"u\"\": null, \"\"v\"\": 1, \"\"sv\"\": [{\"\"b\"\": \"\"8067db44a848ab32c3056a3dbe4edf16\"\", \"\"c\"\": \"\"mBbLR(BvRN1BF^PAFs!B^`U;mA>uOUiFLgDpZXhU#s#%c4wyi&Z7`(d0IxUty-cI#Yp%o~QFF39^sRf>4*EG{zlk;}ArEQ}NQHa9@;T73aPOSTpuh\"\", \"\"m\"\": null, \"\"o\"\": null, \"\"s\"\": \"\"9493d6010fe7845d52149b697729c745\"\", \"\"u\"\": null, \"\"sv\"\": null, \"\"ocf\"\": null, \"\"ocv\"\": null}, {\"\"b\"\": null, \"\"c\"\": \"\"mBbLR(BvRN1BF^PAFs!B^`U;m8QkTKr|h>Q`^NbW(CC|>SD}UM=o%mz(Fw#LQFF39^sRf>4*EG{zlk;}ArEQ}NQHa9@;T73aPOSTpuh\"\", \"\"m\"\": null, \"\"o\"\": null, \"\"s\"\": \"\"b1f0e4bb3855bc33936ef1fddf532765\"\", \"\"u\"\": null, \"\"sv\"\": null, \"\"ocf\"\": null, \"\"ocv\"\": \"\"fbc7a11fc81f2a31c904c5b05572b054824e3b5f5ece78f1b711f93175f0a4a9726157cea247e107\"\"}], \"\"ocf\"\": null, \"\"ocv\"\": null}\")");
         let data_row = DataRow::try_from(&bytes).unwrap();
 
         let col = data_row.columns.first().unwrap();
         let e: EqlEncrypted = col.try_into().unwrap();
-
         let expected = Identifier::new("encrypted", "encrypted_jsonb");
-
         assert_eq!(e.identifier, expected);
 
         // SELECT * FROM encrypted WHERE id = $1;
         // Only encrypted_text is NOT NULL
         let bytes = to_message(b"D\0\0\n\x91\0\n\0\0\0\n1297231342\xff\xff\xff\xff\0\0\nY(\"{\"\"b\"\": null, \"\"c\"\": \"\"mBbJ;S^xMu<v?;UyTSS~VfK;4C(U~uOiKbWSK*!hB3vi!C$luW$k`K6>@++(U20{lxK;qYYaDYF#30N~x;wyOUMoFOB9K!>A_9g9j@+M6V3wENqu#H8gDb9OZewzJaCBv4Uvy=7bie\"\", \"\"i\"\": {\"\"c\"\": \"\"encrypted_text\"\", \"\"t\"\": \"\"encrypted\"\"}, \"\"m\"\": [369, 381, 1758, 403, 35, 609, 1181, 1098, 1347, 1633, 1150, 815, 1997, 234, 1858, 656, 1335, 936, 1204, 630, 1764, 1328, 1649, 1396, 113, 1149, 1499, 1147, 586, 1942, 901, 1256, 1226, 1045, 637, 279, 1162, 1077, 1340, 1336, 1448, 700, 176, 1849, 1915, 1389, 71, 515, 633, 388, 1877, 1339, 1239, 638, 1365, 1380, 1273, 581, 1792, 1716, 145, 512, 814, 272, 1333, 1775, 1572, 1744, 2018, 433, 1641, 1529, 647, 1317, 652, 1606, 1737, 470, 826, 80, 929, 1700, 1619, 1253, 358, 1589, 1971, 1019, 1533, 1624, 573, 1684, 1287, 575, 1761, 527, 404, 1369, 894, 18, 1101, 986, 1772, 1090, 1506, 2015, 1988, 205, 141, 445, 1982], \"\"o\"\": [\"\"faa1f63cb6d36094d1aa50db6c0217eb447a987071119bb127f677b6a7ee0b4fe40eed7cd84e96e8a11bbe3ea14331f3ec4c8f149ce9d2b0253b4676c86557fcec4a5f8ca4e1ee081c66bf0a3cb594c6b5739f77f62fc5e76991869c23a97f01816cde3dfc24b2ca2fbb12b50fde324f18aa51718d681772bf9caf3c059a6748cbcaf4dd1c4fa02645d74699d7d265faf938c339f6cc8f57db9bd4cff8e03cae9e5d21a651b33525e86e335dff61520e8f23d7002f05fa186075a335fb7b2c740133b5a72760ccd216127d69983aa31a090a3b6ca56a48b6372cab60c979465d84dc94e5452c92517b643882fa82c22a26b4feaaa1b0ae8fcb989b10d0351fb3c9c5e56e719f820442612a67fff334438f3f5d35ff6db1b5f7a50670c7fec014f6fc19c352eb011911faf62a230e10c2d16f6c84b46cf9ee7eb1afb9c61a523891e31da2a18b445769d75c11873566dc8196d77e985423226bd1db10e4ce9eb10c2f69db7ce57d47281401617978d2bcfca23b9015b9e705615b8bf773daa87a18417f86e5338a7929fa4f10c6864af09870bfd9ddfb7848\"\", \"\"b41d89a196a35252a965ce3c330eac369ead56e9f06e2016da4d6971fe0b8d6e677e1018e7a1bd2fa0b2c1faaa12650d678352ecc81f6be879213fe78b8004b87dd7dcadec59df4dcafdb3c9aa55dcb2cc2bcf2193574b201c9a1c14764d69716f63b0c1aa30a2846696f2a1c790ca2cb26370d7e20904a8748ea98a95ee3cbb95c5f342de4e71bbf0262e84d59188ea72fe4449a16e7c73f88ed06b9cb724902a85d063c03e9b1a63dd18b9604625ca3cb8110d9c8f93e1771525c51b6ee092d554e84d61df5b557994f32191bb2b6801d9727fb707d5287e6c83d6b16763a6e66526baf80765a58d36df744be7872d2750eb28a86a519a21ee710f618c09cb2bd45f21e805ae4e11eb2987d7be31c32164d4f828fc35c389d516d0d6a54e25041985cffcb6124b4d3fa5b0ba91e19d60e3102370e9c1c768df1b427c682304a1dfdea2d3e514db22057f43d8121b8daf7c434831e5b618bbca9f4e198741927bdc168e4703fb1f703957f7b70491e06bec4adee19d29ef5e938695e1d49ef50ceef0a9c3e46bd8fe309e013e5ea0d35c5ebf3dddd97573\"\"], \"\"s\"\": null, \"\"u\"\": \"\"962d77dfaf892b596b3255c022359e54f3e8dc8b21c3d1b32ebd05555f433192\"\", \"\"v\"\": 1, \"\"sv\"\": null, \"\"ocf\"\": null, \"\"ocv\"\": null}\")\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff");
         let data_row = DataRow::try_from(&bytes).unwrap();
 
-        // let e = DataColumn::parse(bytes).unwrap();
         let col = &data_row.columns[2];
         let e: EqlEncrypted = col.try_into().unwrap();
         let expected = Identifier::new("encrypted", "encrypted_text");
-
         assert_eq!(e.identifier, expected);
     }