Optimize lookup overflow hot paths

Author: ktsaou

.agents/sow/current/SOW-0021-20260613-netipc-at-scale.md

@@ -1156,6 +1156,37 @@ Follow-up mapping:
   - add any remaining broader adversarial tests from the planned matrix beyond the now-covered representative `8192`/`32768` logical-call cases, now-covered mid-logical timeout/abort cases, now-covered malformed follow-up responses after partial progress, now-covered reordered/duplicate response-item corruption, now-covered invalid status/status-dependent/label-table response corruption, now-covered huge valid label isolation cases, now-covered endpoint absence before call, now-covered endpoint disappearance after partial progress, now-covered endpoint disappearance before the first subcall, now-covered zero-item typed lookup calls, now-covered stale request-capacity reconnect cases, now-covered duplicate/unsorted request keys under request splitting, now-covered request cap-minus-one/exact/plus-one boundaries, now-covered exact response-fit plus/minus-one boundaries, now-covered no-progress overflow cases, now-covered raw no-growth overflow cases, now-covered logical response-byte ceilings, now-covered mixed-status cross-language interop cases, and now-covered lookup status codec interop cases;
   - keep lookup-scale interop green across POSIX baseline, POSIX SHM, Windows Named Pipe, and Windows SHM; all four profiles now cover both all-known scale and mixed-status C/Rust/Go directed tests.
 
+## Performance Checkpoint - 2026-06-15
+
+Context:
+
+- A direct benchmark sample showed `cgroups-lookup-mixed-256` at roughly C `61.6k`, Rust `43.7k`, Go `25.9k` requests/s.
+- The gap was not accepted as normal runtime noise. The lookup hot paths were reviewed language-by-language before committing.
+
+Confirmed causes fixed in this checkpoint:
+
+- C, Rust, and Go all had an O(n^2)-style response-overflow suffix-fit check in lookup builders. This was replaced with precomputed suffix bytes for dispatch-owned builders.
+- Rust and Go cgroups dispatch paths called `request.Item(i)` in the payload-exceeded sizing prepass only to recover the already-validated request directory length. Both now read that length directly.
+- Go response validation built item view objects while validating each response item. Go now has validate-only lookup item paths separate from public view construction.
+- Go request views now cache the request packed-start offset instead of recomputing it on each item access.
+
+Focused validation:
+
+- `cd src/go && go test -count=1 ./pkg/netipc/protocol` passed.
+- `cargo test --manifest-path src/crates/netipc/Cargo.toml protocol::lookup -- --test-threads=1` passed.
+- `cmake --build build --target test_protocol -j24 && build/bin/test_protocol` passed with `514 passed, 0 failed`.
+
+Sampled benchmark evidence after fixes, using release benchmark binaries:
+
+- `cgroups-lookup-mixed-256`: C about `63k`, Rust about `48k`, Go about `34k-35k` requests/s.
+- `apps-lookup-mixed-256`: C about `69k`, Rust about `62k`, Go about `40k-42k` requests/s.
+
+Current interpretation:
+
+- Rust is no longer broadly suspicious: `apps-lookup-mixed-256` is close to C, and the cgroups-specific avoidable overhead was reduced.
+- Go remains materially slower than C/Rust in these codec+dispatch microbenchmarks. The obvious avoidable algorithmic and allocation issues found so far are fixed, but the remaining gap still needs full-suite benchmark confirmation and, if required, profiling before declaring it inherent Go/runtime overhead.
+- This checkpoint does not close the performance gate. Full POSIX and Windows benchmark regeneration still needs to be run cleanly after commit/push.
+
 ## Downstream Vendoring Plan
 
 Purpose:

src/crates/netipc/src/protocol/lookup/apps_lookup.rs

@@ -23,6 +23,7 @@ pub const APPS_LOOKUP_KEY_SIZE: usize = 8;
 #[derive(Debug)]
 pub struct AppsLookupRequestView<'a> {
     pub item_count: u32,
+    packed_start: usize,
     payload: &'a [u8],
 }
 
@@ -251,6 +252,7 @@ impl<'a> AppsLookupRequestView<'a> {
         }
         Ok(Self {
             item_count,
+            packed_start: dir_end,
             payload: buf,
         })
     }
@@ -259,12 +261,6 @@ impl<'a> AppsLookupRequestView<'a> {
         if index >= self.item_count {
             return Err(NipcError::OutOfBounds);
         }
-        let dir_size = (self.item_count as usize)
-            .checked_mul(LOOKUP_DIR_ENTRY_SIZE)
-            .ok_or(NipcError::BadItemCount)?;
-        let packed_start = APPS_LOOKUP_REQ_HDR_SIZE
-            .checked_add(dir_size)
-            .ok_or(NipcError::BadItemCount)?;
         let base = APPS_LOOKUP_REQ_HDR_SIZE
             .checked_add(
                 (index as usize)
@@ -274,7 +270,7 @@ impl<'a> AppsLookupRequestView<'a> {
             .ok_or(NipcError::BadItemCount)?;
         let off = u32_at(self.payload, base) as usize;
         Ok(u32_at(
-            checked_subslice(self.payload, packed_start, off, APPS_LOOKUP_KEY_SIZE)?,
+            checked_subslice(self.payload, self.packed_start, off, APPS_LOOKUP_KEY_SIZE)?,
             0,
         ))
     }
@@ -450,7 +446,7 @@ pub struct AppsLookupBuilder<'a> {
     data_offset: usize,
     error: Option<NipcError>,
     payload_exceeded_suffix: bool,
-    payload_exceeded_item_lens: Vec<usize>,
+    payload_exceeded_suffix_bytes: Vec<usize>,
 }
 
 impl<'a> AppsLookupBuilder<'a> {
@@ -471,7 +467,7 @@ impl<'a> AppsLookupBuilder<'a> {
             data_offset,
             error: None,
             payload_exceeded_suffix: false,
-            payload_exceeded_item_lens: Vec::new(),
+            payload_exceeded_suffix_bytes: Vec::new(),
         }
     }
 
@@ -480,7 +476,10 @@ impl<'a> AppsLookupBuilder<'a> {
     }
 
     pub fn set_payload_exceeded_item_lens(&mut self, item_lens: Vec<usize>) {
-        self.payload_exceeded_item_lens = item_lens;
+        match payload_exceeded_suffix_bytes_from_lens(item_lens) {
+            Ok(suffix_bytes) => self.payload_exceeded_suffix_bytes = suffix_bytes,
+            Err(err) => self.error = Some(err),
+        }
     }
 
     #[allow(clippy::too_many_arguments)]
@@ -579,7 +578,7 @@ impl<'a> AppsLookupBuilder<'a> {
         if !payload_exceeded_suffix_fits(
             self.buf.len(),
             item_end,
-            &self.payload_exceeded_item_lens,
+            &self.payload_exceeded_suffix_bytes,
             self.item_count + 1,
             self.max_items,
         ) {
@@ -780,10 +779,14 @@ where
     }
     let mut builder = AppsLookupBuilder::new(resp, request.item_count, 0);
     if request.item_count > 0 {
-        builder.set_payload_exceeded_item_lens(vec![
-            APPS_LOOKUP_ITEM_HDR_SIZE + 3;
-            request.item_count as usize
-        ]);
+        let item_count = request.item_count as usize;
+        let capacity = item_count.checked_add(1).ok_or(NipcError::Overflow)?;
+        let mut item_lens = Vec::with_capacity(capacity);
+        item_lens.resize(item_count, APPS_LOOKUP_ITEM_HDR_SIZE + 3);
+        builder.set_payload_exceeded_item_lens(item_lens);
+        if let Some(err) = builder.error {
+            return Err(err);
+        }
     }
     if !handler(&request, &mut builder) {
         return Err(builder.error().unwrap_or(NipcError::HandlerFailed));

src/crates/netipc/src/protocol/lookup/cgroups_lookup.rs

@@ -16,6 +16,7 @@ pub const CGROUPS_LOOKUP_ITEM_HDR_SIZE: usize = 28;
 #[derive(Debug)]
 pub struct CgroupsLookupRequestView<'a> {
     pub item_count: u32,
+    packed_start: usize,
     payload: &'a [u8],
 }
 
@@ -148,6 +149,7 @@ impl<'a> CgroupsLookupRequestView<'a> {
         }
         Ok(Self {
             item_count,
+            packed_start: dir_end,
             payload: buf,
         })
     }
@@ -156,12 +158,6 @@ impl<'a> CgroupsLookupRequestView<'a> {
         if index >= self.item_count {
             return Err(NipcError::OutOfBounds);
         }
-        let dir_size = (self.item_count as usize)
-            .checked_mul(LOOKUP_DIR_ENTRY_SIZE)
-            .ok_or(NipcError::BadItemCount)?;
-        let packed_start = CGROUPS_LOOKUP_REQ_HDR_SIZE
-            .checked_add(dir_size)
-            .ok_or(NipcError::BadItemCount)?;
         let base = CGROUPS_LOOKUP_REQ_HDR_SIZE
             .checked_add(
                 (index as usize)
@@ -172,10 +168,28 @@ impl<'a> CgroupsLookupRequestView<'a> {
         let off = u32_at(self.payload, base) as usize;
         let len = u32_at(self.payload, base + 4) as usize;
         Ok(StrView {
-            bytes: checked_subslice(self.payload, packed_start, off, len)?,
+            bytes: checked_subslice(self.payload, self.packed_start, off, len)?,
             len: (len - 1) as u32,
         })
     }
+
+    fn payload_exceeded_item_len(&self, index: u32) -> Result<usize, NipcError> {
+        if index >= self.item_count {
+            return Err(NipcError::OutOfBounds);
+        }
+        let base = CGROUPS_LOOKUP_REQ_HDR_SIZE
+            .checked_add(
+                (index as usize)
+                    .checked_mul(LOOKUP_DIR_ENTRY_SIZE)
+                    .ok_or(NipcError::BadItemCount)?,
+            )
+            .ok_or(NipcError::BadItemCount)?;
+        let path_wire_len = u32_at(self.payload, base + 4) as usize;
+        CGROUPS_LOOKUP_ITEM_HDR_SIZE
+            .checked_add(path_wire_len)
+            .and_then(|v| v.checked_add(1))
+            .ok_or(NipcError::Overflow)
+    }
 }
 
 impl<'a> CgroupsLookupResponseView<'a> {
@@ -324,7 +338,7 @@ pub struct CgroupsLookupBuilder<'a> {
     data_offset: usize,
     error: Option<NipcError>,
     payload_exceeded_suffix: bool,
-    payload_exceeded_item_lens: Vec<usize>,
+    payload_exceeded_suffix_bytes: Vec<usize>,
 }
 
 impl<'a> CgroupsLookupBuilder<'a> {
@@ -345,7 +359,7 @@ impl<'a> CgroupsLookupBuilder<'a> {
             data_offset,
             error: None,
             payload_exceeded_suffix: false,
-            payload_exceeded_item_lens: Vec::new(),
+            payload_exceeded_suffix_bytes: Vec::new(),
         }
     }
 
@@ -354,7 +368,10 @@ impl<'a> CgroupsLookupBuilder<'a> {
     }
 
     pub fn set_payload_exceeded_item_lens(&mut self, item_lens: Vec<usize>) {
-        self.payload_exceeded_item_lens = item_lens;
+        match payload_exceeded_suffix_bytes_from_lens(item_lens) {
+            Ok(suffix_bytes) => self.payload_exceeded_suffix_bytes = suffix_bytes,
+            Err(err) => self.error = Some(err),
+        }
     }
 
     pub fn add(
@@ -432,7 +449,7 @@ impl<'a> CgroupsLookupBuilder<'a> {
         if !payload_exceeded_suffix_fits(
             self.buf.len(),
             item_end,
-            &self.payload_exceeded_item_lens,
+            &self.payload_exceeded_suffix_bytes,
             self.item_count + 1,
             self.max_items,
         ) {
@@ -619,16 +636,16 @@ where
     }
     let mut builder = CgroupsLookupBuilder::new(resp, request.item_count, 0);
     if request.item_count > 0 {
-        let mut payload_exceeded_item_lens = Vec::with_capacity(request.item_count as usize);
+        let item_count = request.item_count as usize;
+        let capacity = item_count.checked_add(1).ok_or(NipcError::Overflow)?;
+        let mut payload_exceeded_item_lens = Vec::with_capacity(capacity);
         for i in 0..request.item_count {
-            let item = request.item(i)?;
-            let item_len = CGROUPS_LOOKUP_ITEM_HDR_SIZE
-                .checked_add(item.as_bytes().len())
-                .and_then(|v| v.checked_add(2))
-                .ok_or(NipcError::Overflow)?;
-            payload_exceeded_item_lens.push(item_len);
+            payload_exceeded_item_lens.push(request.payload_exceeded_item_len(i)?);
         }
         builder.set_payload_exceeded_item_lens(payload_exceeded_item_lens);
+        if let Some(err) = builder.error {
+            return Err(err);
+        }
     }
     if !handler(&request, &mut builder) {
         return Err(builder.error().unwrap_or(NipcError::HandlerFailed));

src/crates/netipc/src/protocol/lookup/common.rs

@@ -166,36 +166,49 @@ pub(super) fn lookup_dir_entry_offset(hdr_size: usize, index: u32) -> Result<usi
         .ok_or(NipcError::BadItemCount)
 }
 
+pub(super) fn payload_exceeded_suffix_bytes_from_lens(
+    mut item_lens: Vec<usize>,
+) -> Result<Vec<usize>, NipcError> {
+    item_lens.push(0);
+    for i in (0..item_lens.len() - 1).rev() {
+        let next = item_lens[i + 1];
+        let item_cost = if next > 0 {
+            item_lens[i].checked_add(7).map(|v| v & !7)
+        } else {
+            Some(item_lens[i])
+        }
+        .ok_or(NipcError::Overflow)?;
+        item_lens[i] = item_cost.checked_add(next).ok_or(NipcError::Overflow)?;
+    }
+    Ok(item_lens)
+}
+
 pub(super) fn payload_exceeded_suffix_fits(
     buf_len: usize,
-    mut data_offset: usize,
-    item_lens: &[usize],
+    data_offset: usize,
+    suffix_bytes: &[usize],
     first_index: u32,
     max_items: u32,
 ) -> bool {
-    if item_lens.len() != max_items as usize {
+    let max_items = max_items as usize;
+    let Some(expected_len) = max_items.checked_add(1) else {
+        return true;
+    };
+    if suffix_bytes.len() != expected_len {
         return true;
     }
-    for item_len in item_lens
-        .iter()
-        .take(max_items as usize)
-        .skip(first_index as usize)
-    {
-        let Some(item_start) = data_offset.checked_add(7).map(|v| v & !7) else {
-            return false;
-        };
-        if item_start > buf_len {
-            return false;
-        }
-        let Some(item_end) = item_start.checked_add(*item_len) else {
-            return false;
-        };
-        if item_end > buf_len {
-            return false;
-        }
-        data_offset = item_end;
+    let first_index = first_index as usize;
+    if first_index > max_items {
+        return true;
+    }
+    if data_offset > usize::MAX - 7 {
+        return false;
+    }
+    let item_start = align8(data_offset);
+    if item_start > buf_len {
+        return false;
     }
-    true
+    suffix_bytes[first_index] <= buf_len - item_start
 }
 
 pub(super) fn validate_labels(