Bound delivery delta metadata within poll envelopes

Author: Saga4

crates/locality-protocol/src/freshness_delivery.rs

@@ -21,6 +21,10 @@ use crate::workspace_layout::LayoutDigest;
 pub const FRESHNESS_DELIVERY_READER_VERSION: u16 = 1;
 pub const GENERATION_DELTA_FORMAT_VERSION: u16 = 1;
 pub const MAX_GENERATION_DELTA_ENTRIES: usize = 100_000;
+/// Maximum compact JSON metadata for one complete delta. The transport reserves
+/// the remaining 1 MiB of its 64 MiB poll response for the envelope and terminal
+/// receipt, so V1 never requires metadata pagination.
+pub const MAX_GENERATION_DELTA_METADATA_BYTES: usize = 63 * 1024 * 1024;
 pub const MAX_GENERATION_FILE_BYTES: u64 = 64 * 1024 * 1024;
 pub const MAX_GENERATION_DELTA_CONTENT_BYTES: u64 = 512 * 1024 * 1024;
 pub const MAX_DELIVERY_ID_BYTES: usize = 128;
@@ -455,9 +459,24 @@ impl GenerationDelta {
                 actual: changed_content_bytes,
             });
         }
+        let metadata_bytes = self.serialized_metadata_len()?;
+        if metadata_bytes > MAX_GENERATION_DELTA_METADATA_BYTES {
+            return Err(FreshnessDeliveryError::DeltaMetadataTooLarge {
+                actual: metadata_bytes,
+            });
+        }
         Ok(())
     }
 
+    /// Exact byte length of this delta's compact serde JSON representation.
+    /// Counting writes avoids allocating a second metadata-sized buffer.
+    pub fn serialized_metadata_len(&self) -> Result<usize, FreshnessDeliveryError> {
+        let mut writer = JsonLengthWriter::default();
+        serde_json::to_writer(&mut writer, self)
+            .map_err(|_| FreshnessDeliveryError::CanonicalValueTooLarge)?;
+        Ok(writer.len)
+    }
+
     pub fn canonical_preimage(&self) -> Result<Vec<u8>, FreshnessDeliveryError> {
         self.validate()?;
         let mut output = b"locality.generation-delta.v1\0".to_vec();
@@ -635,6 +654,7 @@ pub enum FreshnessDeliveryError {
     TooManyDeltaEntries { actual: usize },
     FileContentTooLarge { actual: u64 },
     DeltaContentTooLarge { actual: u64 },
+    DeltaMetadataTooLarge { actual: usize },
     NonCanonicalDeltaOrder,
     NonCanonicalTargetInventoryOrder,
     CrossEntryPathReuse,
@@ -702,6 +722,10 @@ impl Display for FreshnessDeliveryError {
                 formatter,
                 "delta contains {actual} content bytes, exceeding {MAX_GENERATION_DELTA_CONTENT_BYTES}"
             ),
+            Self::DeltaMetadataTooLarge { actual } => write!(
+                formatter,
+                "delta metadata encoding is {actual} bytes, exceeding {MAX_GENERATION_DELTA_METADATA_BYTES}"
+            ),
             Self::NonCanonicalDeltaOrder => {
                 formatter.write_str("delta entries are not in canonical order")
             }
@@ -728,6 +752,25 @@ impl Display for FreshnessDeliveryError {
 
 impl std::error::Error for FreshnessDeliveryError {}
 
+#[derive(Default)]
+struct JsonLengthWriter {
+    len: usize,
+}
+
+impl std::io::Write for JsonLengthWriter {
+    fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
+        self.len = self
+            .len
+            .checked_add(bytes.len())
+            .ok_or_else(|| std::io::Error::other("JSON length overflow"))?;
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
 fn validate_versions(
     format_version: u16,
     minimum_reader_version: u16,

crates/locality-protocol/src/freshness_delivery_transport.rs

@@ -16,13 +16,15 @@ use crate::FreshnessEpoch;
 use crate::freshness_delivery::{
     FreshnessReasonCode, FreshnessRetry, GenerationDelta, GenerationDeltaTerminalReceipt,
     GenerationFileIdentity, MAX_DELIVERY_ID_BYTES, MAX_DELIVERY_TIMESTAMP_BYTES,
+    MAX_GENERATION_DELTA_METADATA_BYTES,
 };
 
 pub const GENERATION_TRANSPORT_FORMAT_VERSION: u16 = 1;
 pub const GENERATION_TRANSPORT_READER_VERSION: u16 = 1;
 pub const MAX_GENERATION_TRANSPORT_CAPABILITIES_BYTES: usize = 4 * 1024;
 pub const MAX_GENERATION_TRANSPORT_REQUEST_BYTES: usize = 16 * 1024;
 pub const MAX_GENERATION_DELIVERY_POLL_RESPONSE_BYTES: usize = 64 * 1024 * 1024;
+pub const GENERATION_DELIVERY_POLL_ENVELOPE_HEADROOM_BYTES: usize = 1024 * 1024;
 pub const MAX_GENERATION_BODY_WINDOW_METADATA_BYTES: usize = 16 * 1024;
 pub const MAX_GENERATION_BODY_WINDOW_BYTES: u64 = 4 * 1024 * 1024;
 pub const GENERATION_DELIVERY_POLL_CONTENT_TYPE: &str = "application/json";
@@ -37,6 +39,24 @@ pub const MAX_OPAQUE_DEVICE_SCOPE_ID_BYTES: usize = 128;
 pub const MAX_OPAQUE_PIN_LEASE_ID_BYTES: usize = 256;
 pub const MAX_GENERATION_PIN_OPERATION_ID_BYTES: usize = 128;
 
+const MAX_ESCAPED_DELIVERY_ID_BYTES: usize = 6 * MAX_DELIVERY_ID_BYTES;
+const MAX_ESCAPED_DELIVERY_TIMESTAMP_BYTES: usize = 6 * MAX_DELIVERY_TIMESTAMP_BYTES;
+// Deliberately overcounts V1's identifiers, timestamps, fixed JSON syntax, and
+// numeric/enum fields in addition to the independently bounded capabilities.
+const MAX_GENERATION_DELIVERY_POLL_ENVELOPE_REQUIRED_BYTES: usize =
+    MAX_GENERATION_TRANSPORT_CAPABILITIES_BYTES
+        + 16 * MAX_ESCAPED_DELIVERY_ID_BYTES
+        + 4 * MAX_ESCAPED_DELIVERY_TIMESTAMP_BYTES
+        + 64 * 1024;
+const _: () = assert!(
+    MAX_GENERATION_DELTA_METADATA_BYTES + GENERATION_DELIVERY_POLL_ENVELOPE_HEADROOM_BYTES
+        <= MAX_GENERATION_DELIVERY_POLL_RESPONSE_BYTES
+);
+const _: () = assert!(
+    MAX_GENERATION_DELIVERY_POLL_ENVELOPE_REQUIRED_BYTES
+        <= GENERATION_DELIVERY_POLL_ENVELOPE_HEADROOM_BYTES
+);
+
 pub const GENERATION_TRANSPORT_CAPABILITIES_V1_GOLDEN_JSON: &[u8] =
     include_bytes!("../fixtures/generation-transport-capabilities-v1.json");
 pub const GENERATION_DELIVERY_REQUEST_V1_GOLDEN_JSON: &[u8] =
@@ -367,9 +387,10 @@ impl GenerationDeliveryPollResponse {
             _ => return Err(GenerationTransportContractError::AmbiguousPollResponse),
         }
 
-        let encoded = serde_json::to_vec(self)
-            .expect("serializing a typed generation delivery poll response cannot fail");
-        validate_encoding_length_against(encoded.len(), MAX_GENERATION_DELIVERY_POLL_RESPONSE_BYTES)
+        validate_encoding_length_against(
+            serialized_json_len(self),
+            MAX_GENERATION_DELIVERY_POLL_RESPONSE_BYTES,
+        )
     }
 }
 
@@ -1461,6 +1482,32 @@ fn validate_encoding_length_against(
     }
 }
 
+#[derive(Default)]
+struct JsonLengthWriter {
+    len: usize,
+}
+
+impl std::io::Write for JsonLengthWriter {
+    fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
+        self.len = self
+            .len
+            .checked_add(bytes.len())
+            .ok_or_else(|| std::io::Error::other("JSON length overflow"))?;
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
+fn serialized_json_len(value: &impl Serialize) -> usize {
+    let mut writer = JsonLengthWriter::default();
+    serde_json::to_writer(&mut writer, value)
+        .expect("serializing a typed generation transport value cannot fail");
+    writer.len
+}
+
 fn validate_body_frame_length(actual: usize) -> Result<(), GenerationTransportContractError> {
     let maximum = GENERATION_BODY_WINDOW_METADATA_LENGTH_BYTES
         + MAX_GENERATION_BODY_WINDOW_METADATA_BYTES

crates/locality-protocol/tests/freshness_delivery.rs

@@ -9,9 +9,9 @@ use locality_protocol::freshness_delivery::{
     GENERATION_DELTA_PREIMAGE_V1_GOLDEN_JSON, GENERATION_DELTA_RECEIPT_V1_GOLDEN_JSON,
     GENERATION_DELTA_V1_GOLDEN_JSON, GENERATION_TARGET_INVENTORY_V1_VECTORS_JSON, GenerationDelta,
     GenerationDeltaEntry, GenerationDeltaTerminalReceipt, GenerationFileIdentity,
-    MAX_GENERATION_DELTA_CONTENT_BYTES, MAX_GENERATION_FILE_BYTES, ProviderHealth,
-    ProviderHealthState, ProviderWorkerProgress, PublicationGenerationHealth,
-    PublicationGenerationState, canonical_target_inventory_preimage,
+    MAX_GENERATION_DELTA_CONTENT_BYTES, MAX_GENERATION_DELTA_METADATA_BYTES,
+    MAX_GENERATION_FILE_BYTES, ProviderHealth, ProviderHealthState, ProviderWorkerProgress,
+    PublicationGenerationHealth, PublicationGenerationState, canonical_target_inventory_preimage,
     canonical_target_inventory_sha256,
 };
 use locality_protocol::workspace_layout::LayoutDigest;
@@ -476,3 +476,29 @@ fn empty_delta_advances_generation_and_content_limits_are_bounded() {
         Err(FreshnessDeliveryError::DeltaContentTooLarge { .. })
     ));
 }
+
+#[test]
+fn structurally_valid_delta_rejects_oversized_exact_json_metadata() {
+    let mut oversized = delta();
+    let escaped_id_tail = "\0".repeat(123);
+    let escaped_content_id = "\0".repeat(128);
+    oversized.entries = (0..42_000)
+        .map(|index| GenerationDeltaEntry {
+            old: None,
+            new: Some(identity(
+                &format!("{index:05}{escaped_id_tail}"),
+                &format!("file-{index:05}"),
+                &escaped_content_id,
+                '3',
+                1,
+            )),
+        })
+        .collect();
+
+    let actual = oversized.serialized_metadata_len().unwrap();
+    assert!(actual > MAX_GENERATION_DELTA_METADATA_BYTES);
+    assert_eq!(
+        oversized.validate(),
+        Err(FreshnessDeliveryError::DeltaMetadataTooLarge { actual })
+    );
+}

crates/locality-protocol/tests/freshness_delivery_transport.rs

@@ -3,12 +3,14 @@ use locality_core::portable::{
 };
 use locality_core::workspace_layout::PortableMountId;
 use locality_protocol::freshness_delivery::{
-    GENERATION_DELTA_RECEIPT_V1_GOLDEN_JSON, GenerationDeltaTerminalReceipt, GenerationFileIdentity,
+    GENERATION_DELTA_RECEIPT_V1_GOLDEN_JSON, GenerationDeltaTerminalReceipt,
+    GenerationFileIdentity, MAX_GENERATION_DELTA_METADATA_BYTES,
 };
 use locality_protocol::freshness_delivery_transport::{
     GENERATION_BODY_WINDOW_CONTENT_TYPE, GENERATION_BODY_WINDOW_FRAME_V1_GOLDEN_HEX,
     GENERATION_BODY_WINDOW_METADATA_V1_GOLDEN_JSON, GENERATION_BODY_WINDOW_REQUEST_V1_GOLDEN_JSON,
-    GENERATION_DELIVERY_ACKNOWLEDGMENT_V1_GOLDEN_JSON, GENERATION_DELIVERY_POLL_V1_GOLDEN_JSON,
+    GENERATION_DELIVERY_ACKNOWLEDGMENT_V1_GOLDEN_JSON,
+    GENERATION_DELIVERY_POLL_ENVELOPE_HEADROOM_BYTES, GENERATION_DELIVERY_POLL_V1_GOLDEN_JSON,
     GENERATION_DELIVERY_REQUEST_V1_GOLDEN_JSON, GENERATION_PIN_LEASE_V1_GOLDEN_JSON,
     GENERATION_TRANSPORT_CAPABILITIES_V1_GOLDEN_JSON, GENERATION_TRANSPORT_FORMAT_VERSION,
     GENERATION_TRANSPORT_READER_VERSION, GenerationBodyRange, GenerationBodyWindowCapability,
@@ -247,11 +249,22 @@ fn poll_envelope_statuses_are_bounded_and_bound_to_the_request() {
         })
     );
 
+    let normal_delivery = serde_json::to_vec(&polls.delivery).unwrap();
+    assert!(normal_delivery.len() < MAX_GENERATION_DELIVERY_POLL_RESPONSE_BYTES);
+    GenerationDeliveryPollResponse::decode_json(&normal_delivery, &request)
+        .expect("normal delivery poll");
+
+    assert_eq!(
+        MAX_GENERATION_DELTA_METADATA_BYTES + GENERATION_DELIVERY_POLL_ENVELOPE_HEADROOM_BYTES,
+        MAX_GENERATION_DELIVERY_POLL_RESPONSE_BYTES
+    );
+    let mut boundary = serde_json::to_vec(&polls.no_delivery).unwrap();
+    boundary.resize(MAX_GENERATION_DELIVERY_POLL_RESPONSE_BYTES, b' ');
+    GenerationDeliveryPollResponse::decode_json(&boundary, &request)
+        .expect("poll at exact response boundary");
+    boundary.push(b' ');
     assert!(matches!(
-        GenerationDeliveryPollResponse::decode_json(
-            &vec![b' '; MAX_GENERATION_DELIVERY_POLL_RESPONSE_BYTES + 1],
-            &request,
-        ),
+        GenerationDeliveryPollResponse::decode_json(&boundary, &request),
         Err(GenerationTransportContractError::EncodingTooLarge { .. })
     ));
 }

docs/freshness-delivery-transport.md

@@ -27,7 +27,12 @@ adapter maps that portable value to its local daemon result after authentication
 the protocol value contains no reader or stream type.
 
 The poll response uses `Content-Type: application/json`, is capped at 64 MiB,
-and has exactly one of these status/payload combinations:
+and reserves 1 MiB for its bounded envelope and terminal receipt. A complete
+delta's exact compact serde JSON is capped at 63 MiB, so every delta accepted by
+the public contract fits in one V1 poll response without metadata pagination.
+The entry-count ceiling remains 100,000. Delta validation counts serializer
+output without allocating a second metadata-sized buffer. The response has
+exactly one of these status/payload combinations:
 
 - `delivery` has one delta and terminal receipt and no error.
 - `no_delivery` has neither a delivery nor an error.
@@ -187,6 +192,8 @@ is integrated with generation selection.
 | --- | ---: |
 | Capability JSON | 4 KiB |
 | Request/metadata JSON | 16 KiB |
+| Complete delta metadata JSON | 63 MiB |
+| Poll envelope and terminal-receipt headroom | 1 MiB |
 | Poll response JSON | 64 MiB |
 | Body window | 4 MiB |
 | Body-window frame | 4-byte prefix + 16 KiB metadata + 4 MiB body |