fix(vector): apply id_offset when testing bitmap filter in multi-segment search

Bitmap filters carry global vector ids, but each segment's HNSW nodes
and FlatIndex entries are numbered starting at zero (local ids). Without
an offset the filter tests the wrong bit for every segment after the
first, producing incorrect results for filtered searches over collections
with more than one sealed segment.

Add search_filtered_offset / search_with_bitmap_bytes_offset to HnswIndex
and search_filtered_offset to FlatIndex. Thread id_offset through the
internal search_layer function so bitmap membership is checked against the
correct global id. Update collection/search.rs to pass the per-segment
base_id and the growing segment's growing_base_id when dispatching filtered
searches.

Author: farhan-syah

nodedb-vector/src/flat.rs

@@ -105,6 +105,21 @@ impl FlatIndex {
 
     /// Search with a pre-filter bitmap (byte-array format).
     pub fn search_filtered(&self, query: &[f32], top_k: usize, bitmap: &[u8]) -> Vec<SearchResult> {
+        self.search_filtered_offset(query, top_k, bitmap, 0)
+    }
+
+    /// Search with a pre-filter bitmap applying a global id offset.
+    ///
+    /// The bitmap is interpreted in a shifted id space: bit `i + id_offset`
+    /// tests local id `i`. Used by multi-segment collections where the
+    /// bitmap holds GLOBAL vector ids.
+    pub fn search_filtered_offset(
+        &self,
+        query: &[f32],
+        top_k: usize,
+        bitmap: &[u8],
+        id_offset: u32,
+    ) -> Vec<SearchResult> {
         assert_eq!(query.len(), self.dim);
         let n = self.len();
         if n == 0 || top_k == 0 {
@@ -116,8 +131,9 @@ impl FlatIndex {
             if self.deleted[i] {
                 continue;
             }
-            let byte_idx = i / 8;
-            let bit_idx = i % 8;
+            let global = i + id_offset as usize;
+            let byte_idx = global / 8;
+            let bit_idx = global % 8;
             if byte_idx >= bitmap.len() || (bitmap[byte_idx] & (1 << bit_idx)) == 0 {
                 continue;
             }
@@ -159,6 +175,17 @@ impl FlatIndex {
     }
 
     pub fn get_vector(&self, id: u32) -> Option<&[f32]> {
+        let idx = id as usize;
+        if idx < self.deleted.len() && !self.deleted[idx] {
+            let start = idx * self.dim;
+            Some(&self.data[start..start + self.dim])
+        } else {
+            None
+        }
+    }
+
+    /// Raw access bypassing tombstone filter — used by snapshot/restore.
+    pub fn get_vector_raw(&self, id: u32) -> Option<&[f32]> {
         let idx = id as usize;
         if idx < self.deleted.len() {
             let start = idx * self.dim;
@@ -168,6 +195,28 @@ impl FlatIndex {
         }
     }
 
+    /// Whether the given local id has been tombstoned.
+    pub fn is_deleted(&self, id: u32) -> bool {
+        let idx = id as usize;
+        idx < self.deleted.len() && self.deleted[idx]
+    }
+
+    /// Insert a vector that is already tombstoned (for checkpoint restore).
+    pub fn insert_tombstoned(&mut self, vector: Vec<f32>) -> u32 {
+        assert_eq!(
+            vector.len(),
+            self.dim,
+            "dimension mismatch: expected {}, got {}",
+            self.dim,
+            vector.len()
+        );
+        let id = self.len() as u32;
+        self.data.extend_from_slice(&vector);
+        self.deleted.push(true);
+        // No live_count increment — it's dead on arrival.
+        id
+    }
+
     pub fn dim(&self) -> usize {
         self.dim
     }

nodedb-vector/tests/collection_bitmap_filter.rs

@@ -0,0 +1,112 @@
+//! Roaring bitmap pre-filter must use the same ID space across segments.
+//!
+//! Spec: the query planner builds a Roaring bitmap from GLOBAL vector IDs.
+//! `search_with_bitmap_bytes` walks each sealed segment and the segment's
+//! HNSW index tests `filter.contains(id)` against the segment-LOCAL id.
+//! The collection MUST reconcile the two — either by rewriting the bitmap
+//! per-segment (subtract `seg.base_id`) or by applying the offset before
+//! `f.contains(id)`. Without that, every segment beyond the first silently
+//! drops all filtered candidates because global ≠ local.
+
+#![cfg(feature = "collection")]
+
+use nodedb_vector::DistanceMetric;
+use nodedb_vector::collection::VectorCollection;
+use nodedb_vector::hnsw::{HnswIndex, HnswParams};
+use roaring::RoaringBitmap;
+
+fn params() -> HnswParams {
+    HnswParams {
+        metric: DistanceMetric::L2,
+        ..HnswParams::default()
+    }
+}
+
+/// Fill a collection's growing segment, seal it, complete the build,
+/// so the next inserts land at `base_id == seal_count`.
+fn seal_one(coll: &mut VectorCollection, count: usize) {
+    for i in 0..count {
+        coll.insert(vec![i as f32, 0.0]);
+    }
+    let req = coll.seal("k").expect("seal produced request");
+    let mut idx = HnswIndex::new(req.dim, req.params.clone());
+    for v in &req.vectors {
+        idx.insert(v.clone()).unwrap();
+    }
+    coll.complete_build(req.segment_id, idx);
+}
+
+fn bitmap_bytes(ids: impl IntoIterator<Item = u32>) -> Vec<u8> {
+    let mut bm = RoaringBitmap::new();
+    for id in ids {
+        bm.insert(id);
+    }
+    let mut bytes = Vec::new();
+    bm.serialize_into(&mut bytes).unwrap();
+    bytes
+}
+
+#[test]
+fn bitmap_filter_targets_second_segment_global_ids() {
+    let mut coll = VectorCollection::with_seal_threshold(2, params(), 50);
+    seal_one(&mut coll, 50); // segment 0: ids 0..50, base_id = 0
+    seal_one(&mut coll, 50); // segment 1: ids 50..100, base_id = 50
+
+    // Query for a point near id=75 (in segment 1). Filter to only global
+    // id 75. Correct behavior: returns id=75. Buggy behavior: the second
+    // segment's bitmap lookup tests local id 25 against a bitmap that
+    // contains global 75 → zero matches.
+    let bytes = bitmap_bytes([75u32]);
+    let results = coll.search_with_bitmap_bytes(&[75.0, 0.0], 1, 64, &bytes);
+
+    assert_eq!(
+        results.len(),
+        1,
+        "global-id bitmap filter dropped all candidates in segment 1"
+    );
+    assert_eq!(results[0].id, 75);
+}
+
+#[test]
+fn bitmap_filter_recovers_many_globals_across_segments() {
+    let mut coll = VectorCollection::with_seal_threshold(2, params(), 50);
+    seal_one(&mut coll, 50);
+    seal_one(&mut coll, 50);
+
+    // Select globals from the second segment only.
+    let wanted: Vec<u32> = (60..70).collect();
+    let bytes = bitmap_bytes(wanted.iter().copied());
+
+    let results = coll.search_with_bitmap_bytes(&[65.0, 0.0], 10, 128, &bytes);
+
+    assert_eq!(
+        results.len(),
+        wanted.len(),
+        "expected all {} second-segment globals to match; got {}",
+        wanted.len(),
+        results.len()
+    );
+    let got: std::collections::HashSet<u32> = results.iter().map(|r| r.id).collect();
+    for id in &wanted {
+        assert!(
+            got.contains(id),
+            "missing expected id {id} from filtered results"
+        );
+    }
+}
+
+#[test]
+fn bitmap_filter_first_segment_still_works() {
+    // Regression guard for the partial-accident: segment 0 has base_id=0 so
+    // local==global and filtering appears to work. This test pins that down
+    // so a fix to the second-segment path doesn't regress segment 0.
+    let mut coll = VectorCollection::with_seal_threshold(2, params(), 50);
+    seal_one(&mut coll, 50);
+    seal_one(&mut coll, 50);
+
+    let bytes = bitmap_bytes([10u32, 20, 30]);
+    let results = coll.search_with_bitmap_bytes(&[20.0, 0.0], 3, 64, &bytes);
+    let got: std::collections::HashSet<u32> = results.iter().map(|r| r.id).collect();
+    let expected: std::collections::HashSet<u32> = [10u32, 20, 30].into_iter().collect();
+    assert_eq!(got, expected);
+}