feat(api): compress large data revisions with gzip

Author: azasypkin

Cargo.lock

@@ -30,6 +30,7 @@ csv = "1.4.0"
 deno_core = "0.394.0"
 dotenvy = "0.15.7"
 figment = "0.10.19"
+flate2 = "1.1.9"
 futures = "0.3.32"
 handlebars = "6.4.0"
 http = "1.4.0"
@@ -96,6 +97,7 @@ default = [
     "lettre/tokio1-rustls",
     "mediatype/serde",
     "postcard/use-std",
+    "reqwest/gzip",
     "reqwest/http2",
     "reqwest/json",
     "reqwest-middleware/json",

Cargo.toml

@@ -0,0 +1,5 @@
+-- Switch trackers_data.data to EXTERNAL storage since the application now handles
+-- compression (gzip) before writing. EXTERNAL tells PostgreSQL to store the value
+-- out-of-line (TOAST) when large but skip its own pglz compression pass, avoiding
+-- double-compression overhead.
+ALTER TABLE trackers_data ALTER COLUMN data SET STORAGE EXTERNAL;

migrations/20260406000000_trackers_data_external_storage.sql

@@ -94,6 +94,42 @@ pub mod tests {
 
     pub use super::{dns_resolver::tests::*, smtp::tests::*};
 
+    /// Gzip-compresses the given bytes.
+    fn gzip_encode(data: &[u8]) -> Vec<u8> {
+        use flate2::{Compression, write::GzEncoder};
+        use std::io::Write;
+        let mut encoder = GzEncoder::new(Vec::new(), Compression::fast());
+        encoder.write_all(data).unwrap();
+        encoder.finish().unwrap()
+    }
+
+    #[tokio::test]
+    async fn http_client_negotiates_gzip_compression() -> anyhow::Result<()> {
+        let server = httpmock::MockServer::start();
+        let body = serde_json::json!({"message": "hello from gzip"});
+        let body_bytes = serde_json::to_vec(&body)?;
+        let compressed = gzip_encode(&body_bytes);
+
+        let mock = server.mock(|when, then| {
+            when.method(httpmock::Method::GET)
+                .path("/gzip-test")
+                .header_exists("accept-encoding");
+            then.status(200)
+                .header("Content-Type", "application/json")
+                .header("Content-Encoding", "gzip")
+                .body(compressed);
+        });
+
+        let client = reqwest::Client::builder().build()?;
+        let response = client.get(server.url("/gzip-test")).send().await?;
+        let json: serde_json::Value = response.json().await?;
+
+        mock.assert();
+        assert_eq!(json, body);
+
+        Ok(())
+    }
+
     #[tokio::test]
     async fn correctly_checks_public_web_urls() -> anyhow::Result<()> {
         let public_network = Network {

src/network.rs

@@ -1,9 +1,23 @@
+use flate2::{Compression, read::GzDecoder, write::GzEncoder};
 use retrack_types::trackers::{TrackerDataRevision, TrackerDataValue};
 use serde_json::Value as JsonValue;
-use std::collections::VecDeque;
+use std::{
+    collections::VecDeque,
+    io::{Read, Write},
+};
 use time::OffsetDateTime;
 use uuid::Uuid;
 
+/// Minimum size (in bytes) of Postcard-encoded data before gzip compression is applied.
+/// Values smaller than this typically don't benefit from compression.
+const COMPRESSION_THRESHOLD: usize = 512;
+
+/// Magic byte prefixed to gzip-compressed data blobs to distinguish them from legacy
+/// uncompressed Postcard data. Uncompressed Postcard blobs never start with this byte
+/// because the first Postcard byte is a varint-encoded Vec length (0x00 would mean an
+/// empty vec, which is rejected on read anyway).
+const GZIP_MAGIC: u8 = 0x00;
+
 #[derive(Debug, Eq, PartialEq, Clone)]
 pub(super) struct RawTrackerDataRevision {
     pub id: Uuid,
@@ -12,11 +26,41 @@ pub(super) struct RawTrackerDataRevision {
     pub created_at: OffsetDateTime,
 }
 
+fn decompress_data(data: &[u8]) -> anyhow::Result<Vec<u8>> {
+    if data.first() == Some(&GZIP_MAGIC) {
+        let mut decoder = GzDecoder::new(&data[1..]);
+        let mut decompressed = Vec::new();
+        decoder.read_to_end(&mut decompressed)?;
+        Ok(decompressed)
+    } else {
+        Ok(data.to_vec())
+    }
+}
+
+fn compress_data(postcard_bytes: Vec<u8>) -> anyhow::Result<Vec<u8>> {
+    if postcard_bytes.len() < COMPRESSION_THRESHOLD {
+        return Ok(postcard_bytes);
+    }
+
+    let mut buf = Vec::with_capacity(postcard_bytes.len() / 2);
+    buf.push(GZIP_MAGIC);
+    let mut encoder = GzEncoder::new(&mut buf, Compression::default());
+    encoder.write_all(&postcard_bytes)?;
+    encoder.finish()?;
+
+    if buf.len() < postcard_bytes.len() {
+        Ok(buf)
+    } else {
+        Ok(postcard_bytes)
+    }
+}
+
 impl TryFrom<RawTrackerDataRevision> for TrackerDataRevision {
     type Error = anyhow::Error;
 
     fn try_from(raw: RawTrackerDataRevision) -> Result<Self, Self::Error> {
-        let mut original_and_mods = postcard::from_bytes::<Vec<String>>(&raw.data)?
+        let postcard_bytes = decompress_data(&raw.data)?;
+        let mut original_and_mods = postcard::from_bytes::<Vec<String>>(&postcard_bytes)?
             .into_iter()
             .map(|raw_value| serde_json::from_str(&raw_value))
             .collect::<Result<VecDeque<JsonValue>, _>>()?;
@@ -41,15 +85,16 @@ impl TryFrom<&TrackerDataRevision> for RawTrackerDataRevision {
     type Error = anyhow::Error;
 
     fn try_from(item: &TrackerDataRevision) -> Result<Self, Self::Error> {
+        let postcard_bytes = postcard::to_stdvec(
+            &(&item.data)
+                .into_iter()
+                .map(|value| value.to_string())
+                .collect::<Vec<_>>(),
+        )?;
         Ok(Self {
             id: item.id,
             tracker_id: item.tracker_id,
-            data: postcard::to_stdvec(
-                &(&item.data)
-                    .into_iter()
-                    .map(|value| value.to_string())
-                    .collect::<Vec<_>>(),
-            )?,
+            data: compress_data(postcard_bytes)?,
             created_at: item.created_at,
         })
     }
@@ -90,4 +135,82 @@ mod tests {
 
         Ok(())
     }
+
+    #[test]
+    fn small_data_is_not_compressed() -> anyhow::Result<()> {
+        let data_revision = TrackerDataRevision {
+            id: uuid!("00000000-0000-0000-0000-000000000001"),
+            tracker_id: uuid!("00000000-0000-0000-0000-000000000002"),
+            data: TrackerDataValue::new(json!("small")),
+            created_at: OffsetDateTime::from_unix_timestamp(946720800)?,
+        };
+
+        let raw = RawTrackerDataRevision::try_from(&data_revision)?;
+        assert_ne!(raw.data.first(), Some(&super::GZIP_MAGIC));
+        assert_eq!(TrackerDataRevision::try_from(raw)?, data_revision);
+
+        Ok(())
+    }
+
+    #[test]
+    fn large_data_is_compressed_and_decompressed() -> anyhow::Result<()> {
+        let large_json = json!({
+            "content": "x".repeat(2000),
+            "nested": { "array": (0..100).collect::<Vec<_>>() }
+        });
+        let data_revision = TrackerDataRevision {
+            id: uuid!("00000000-0000-0000-0000-000000000001"),
+            tracker_id: uuid!("00000000-0000-0000-0000-000000000002"),
+            data: TrackerDataValue::new(large_json),
+            created_at: OffsetDateTime::from_unix_timestamp(946720800)?,
+        };
+
+        let raw = RawTrackerDataRevision::try_from(&data_revision)?;
+        assert_eq!(raw.data.first(), Some(&super::GZIP_MAGIC));
+
+        let postcard_bytes = postcard::to_stdvec(
+            &(&data_revision.data)
+                .into_iter()
+                .map(|v| v.to_string())
+                .collect::<Vec<_>>(),
+        )?;
+        assert!(
+            raw.data.len() < postcard_bytes.len(),
+            "compressed ({}) should be smaller than uncompressed ({})",
+            raw.data.len(),
+            postcard_bytes.len()
+        );
+
+        assert_eq!(TrackerDataRevision::try_from(raw)?, data_revision);
+
+        Ok(())
+    }
+
+    #[test]
+    fn can_read_legacy_uncompressed_data() -> anyhow::Result<()> {
+        let data_revision = TrackerDataRevision {
+            id: uuid!("00000000-0000-0000-0000-000000000001"),
+            tracker_id: uuid!("00000000-0000-0000-0000-000000000002"),
+            data: TrackerDataValue::new(json!({"key": "value".repeat(200)})),
+            created_at: OffsetDateTime::from_unix_timestamp(946720800)?,
+        };
+
+        let postcard_bytes = postcard::to_stdvec(
+            &(&data_revision.data)
+                .into_iter()
+                .map(|v| v.to_string())
+                .collect::<Vec<_>>(),
+        )?;
+
+        let raw = RawTrackerDataRevision {
+            id: data_revision.id,
+            tracker_id: data_revision.tracker_id,
+            data: postcard_bytes,
+            created_at: data_revision.created_at,
+        };
+
+        assert_eq!(TrackerDataRevision::try_from(raw)?, data_revision);
+
+        Ok(())
+    }
 }