Reserve lookup overflow suffix capacity

Author: ktsaou

.agents/sow/current/SOW-0021-20260613-netipc-at-scale.md

@@ -1037,6 +1037,13 @@ Tests or equivalent validation:
   - C Windows now validates the same no-progress `PAYLOAD_EXCEEDED` rejection for APPS_LOOKUP and CGROUPS_LOOKUP.
   - Go POSIX/Windows, Rust POSIX/Windows, and C POSIX/Windows validate logical response-byte ceilings for APPS_LOOKUP and CGROUPS_LOOKUP.
   - C POSIX also validates a final stitched response-buffer allocation fault; Go and Rust use explicit logical response-byte ceilings as the deterministic memory-pressure simulation available in those test harnesses.
+- Large response split/stitch suffix-reservation validation:
+  - Added C, Rust, and Go POSIX/Windows tests where one logical APPS_LOOKUP or CGROUPS_LOOKUP request receives large labeled known items that cannot fit in one response payload, requiring transparent `PAYLOAD_EXCEEDED` suffix retries and final response stitching.
+  - The first C POSIX fail-first run exposed a real builder gap: lookup builders could accept one more full known item and then have too little buffer left to encode `PAYLOAD_EXCEEDED` for the remaining suffix. The fix reserves space for the compact overflow suffix before committing another full item.
+  - C protocol/server dispatch now provides per-request compact suffix item lengths to APPS_LOOKUP and CGROUPS_LOOKUP builders; Go and Rust use the same reservation model in both protocol dispatch helpers and raw service dispatchers.
+  - Focused POSIX validation passed: `cmake --build build-coverage --target test_service -j12 && /usr/bin/ctest --test-dir build-coverage --output-on-failure -R '^test_service$'`; `cd src/go && go test -count=1 -timeout=300s ./pkg/netipc/service/raw -run '^TestLookupLargeResponseSplit$'`; `cargo test --manifest-path src/crates/netipc/Cargo.toml large_response_split -- --nocapture`.
+  - Focused Windows validation passed on `win11`: `NIPC_TEST_FILTER=large_response_split timeout 600 build-windows-focused/bin/test_win_service_extra.exe`; `cd src/go && "/c/Program Files/Go/bin/go.exe" test -count=1 -timeout=300s ./pkg/netipc/service/raw -run "^TestWinLookupLargeResponseSplit$"`; `/c/Users/costa/.cargo/bin/cargo.exe test --manifest-path src/crates/netipc/Cargo.toml large_response_split -- --nocapture`.
+  - Protocol-level validation also passed: C `test_protocol`, Go `./pkg/netipc/protocol`, and Rust `protocol::lookup` tests.
 - Note: `ctest` on `$PATH` resolved to a broken local Python wrapper missing the `cmake` module; validation used `/usr/bin/ctest`.
 
 Real-use evidence:

src/crates/netipc/src/protocol/lookup/apps_lookup.rs

@@ -450,6 +450,7 @@ pub struct AppsLookupBuilder<'a> {
     data_offset: usize,
     error: Option<NipcError>,
     payload_exceeded_suffix: bool,
+    payload_exceeded_item_lens: Vec<usize>,
 }
 
 impl<'a> AppsLookupBuilder<'a> {
@@ -470,13 +471,18 @@ impl<'a> AppsLookupBuilder<'a> {
             data_offset,
             error: None,
             payload_exceeded_suffix: false,
+            payload_exceeded_item_lens: Vec::new(),
         }
     }
 
     pub fn set_generation(&mut self, generation: u64) {
         self.generation = generation;
     }
 
+    pub fn set_payload_exceeded_item_lens(&mut self, item_lens: Vec<usize>) {
+        self.payload_exceeded_item_lens = item_lens;
+    }
+
     #[allow(clippy::too_many_arguments)]
     pub fn add(
         &mut self,
@@ -570,6 +576,15 @@ impl<'a> AppsLookupBuilder<'a> {
             Some(v) if v <= self.buf.len() => v,
             _ => return self.note_item_overflow(pid),
         };
+        if !payload_exceeded_suffix_fits(
+            self.buf.len(),
+            item_end,
+            &self.payload_exceeded_item_lens,
+            self.item_count + 1,
+            self.max_items,
+        ) {
+            return self.note_item_overflow(pid);
+        }
         if item_start > self.data_offset {
             self.buf[self.data_offset..item_start].fill(0);
         }
@@ -764,6 +779,12 @@ where
         return Err(NipcError::Overflow);
     }
     let mut builder = AppsLookupBuilder::new(resp, request.item_count, 0);
+    if request.item_count > 0 {
+        builder.set_payload_exceeded_item_lens(vec![
+            APPS_LOOKUP_ITEM_HDR_SIZE + 3;
+            request.item_count as usize
+        ]);
+    }
     if !handler(&request, &mut builder) {
         return Err(builder.error().unwrap_or(NipcError::BadLayout));
     }

src/crates/netipc/src/protocol/lookup/cgroups_lookup.rs

@@ -324,6 +324,7 @@ pub struct CgroupsLookupBuilder<'a> {
     data_offset: usize,
     error: Option<NipcError>,
     payload_exceeded_suffix: bool,
+    payload_exceeded_item_lens: Vec<usize>,
 }
 
 impl<'a> CgroupsLookupBuilder<'a> {
@@ -344,13 +345,18 @@ impl<'a> CgroupsLookupBuilder<'a> {
             data_offset,
             error: None,
             payload_exceeded_suffix: false,
+            payload_exceeded_item_lens: Vec::new(),
         }
     }
 
     pub fn set_generation(&mut self, generation: u64) {
         self.generation = generation;
     }
 
+    pub fn set_payload_exceeded_item_lens(&mut self, item_lens: Vec<usize>) {
+        self.payload_exceeded_item_lens = item_lens;
+    }
+
     pub fn add(
         &mut self,
         status: u16,
@@ -423,6 +429,15 @@ impl<'a> CgroupsLookupBuilder<'a> {
             Some(v) if v <= self.buf.len() => v,
             _ => return self.note_item_overflow(path),
         };
+        if !payload_exceeded_suffix_fits(
+            self.buf.len(),
+            item_end,
+            &self.payload_exceeded_item_lens,
+            self.item_count + 1,
+            self.max_items,
+        ) {
+            return self.note_item_overflow(path);
+        }
         if item_start > self.data_offset {
             self.buf[self.data_offset..item_start].fill(0);
         }
@@ -603,6 +618,18 @@ where
         return Err(NipcError::Overflow);
     }
     let mut builder = CgroupsLookupBuilder::new(resp, request.item_count, 0);
+    if request.item_count > 0 {
+        let mut payload_exceeded_item_lens = Vec::with_capacity(request.item_count as usize);
+        for i in 0..request.item_count {
+            let item = request.item(i)?;
+            let item_len = CGROUPS_LOOKUP_ITEM_HDR_SIZE
+                .checked_add(item.as_bytes().len())
+                .and_then(|v| v.checked_add(2))
+                .ok_or(NipcError::Overflow)?;
+            payload_exceeded_item_lens.push(item_len);
+        }
+        builder.set_payload_exceeded_item_lens(payload_exceeded_item_lens);
+    }
     if !handler(&request, &mut builder) {
         return Err(builder.error().unwrap_or(NipcError::BadLayout));
     }

src/crates/netipc/src/protocol/lookup/common.rs

@@ -166,6 +166,38 @@ pub(super) fn lookup_dir_entry_offset(hdr_size: usize, index: u32) -> Result<usi
         .ok_or(NipcError::BadItemCount)
 }
 
+pub(super) fn payload_exceeded_suffix_fits(
+    buf_len: usize,
+    mut data_offset: usize,
+    item_lens: &[usize],
+    first_index: u32,
+    max_items: u32,
+) -> bool {
+    if item_lens.len() != max_items as usize {
+        return true;
+    }
+    for item_len in item_lens
+        .iter()
+        .take(max_items as usize)
+        .skip(first_index as usize)
+    {
+        let Some(item_start) = data_offset.checked_add(7).map(|v| v & !7) else {
+            return false;
+        };
+        if item_start > buf_len {
+            return false;
+        }
+        let Some(item_end) = item_start.checked_add(*item_len) else {
+            return false;
+        };
+        if item_end > buf_len {
+            return false;
+        }
+        data_offset = item_end;
+    }
+    true
+}
+
 pub(super) fn validate_labels(
     item: &[u8],
     hdr_size: usize,

src/crates/netipc/src/service/raw/apps_lookup.rs

@@ -3,8 +3,9 @@ use super::common::lookup_raw_response_size;
 use super::dispatch::{DispatchError, DispatchHandler};
 use crate::protocol::{
     self, AppsLookupBuilder, AppsLookupRequestView, AppsLookupResponseView, NipcError,
-    APPS_LOOKUP_KEY_SIZE, APPS_LOOKUP_REQ_HDR_SIZE, APPS_LOOKUP_RESP_HDR_SIZE,
-    LOOKUP_DIR_ENTRY_SIZE, METHOD_APPS_LOOKUP, PID_LOOKUP_PAYLOAD_EXCEEDED,
+    APPS_LOOKUP_ITEM_HDR_SIZE, APPS_LOOKUP_KEY_SIZE, APPS_LOOKUP_REQ_HDR_SIZE,
+    APPS_LOOKUP_RESP_HDR_SIZE, LOOKUP_DIR_ENTRY_SIZE, METHOD_APPS_LOOKUP,
+    PID_LOOKUP_PAYLOAD_EXCEEDED,
 };
 use std::sync::Arc;
 
@@ -193,6 +194,12 @@ pub fn apps_lookup_dispatch(handler: AppsLookupHandler) -> DispatchHandler {
             return Err(DispatchError::Overflow);
         }
         let mut builder = AppsLookupBuilder::new(response_buf, request.item_count, 0);
+        if request.item_count > 0 {
+            builder.set_payload_exceeded_item_lens(vec![
+                APPS_LOOKUP_ITEM_HDR_SIZE + 3;
+                request.item_count as usize
+            ]);
+        }
         if !handler(&request, &mut builder) {
             return Err(DispatchError::HandlerFailed);
         }

src/crates/netipc/src/service/raw/cgroups_lookup.rs

@@ -245,6 +245,18 @@ pub fn cgroups_lookup_dispatch(handler: CgroupsLookupHandler) -> DispatchHandler
             return Err(DispatchError::Overflow);
         }
         let mut builder = CgroupsLookupBuilder::new(response_buf, request.item_count, 0);
+        if request.item_count > 0 {
+            let mut payload_exceeded_item_lens = Vec::with_capacity(request.item_count as usize);
+            for i in 0..request.item_count {
+                let item = request.item(i).map_err(|_| DispatchError::BadEnvelope)?;
+                let item_len = CGROUPS_LOOKUP_ITEM_HDR_SIZE
+                    .checked_add(item.as_bytes().len())
+                    .and_then(|v| v.checked_add(2))
+                    .ok_or(DispatchError::Overflow)?;
+                payload_exceeded_item_lens.push(item_len);
+            }
+            builder.set_payload_exceeded_item_lens(payload_exceeded_item_lens);
+        }
         if !handler(&request, &mut builder) {
             return Err(DispatchError::HandlerFailed);
         }