fix(vector): cap HNSW layer assignment at MAX_LAYER_CAP (16)

random_layer() could theoretically produce very large values for
unlucky RNG draws, promoting max_layer to an unbounded height and
making every subsequent search's Phase-1 greedy descent O(max_layer).
Apply a hard cap of 16 layers — standard practice for production HNSW
deployments. Also refactor compact() to expose compact_with_map()
returning the old→new id remapping needed by doc_id_map maintenance.

Author: farhan-syah

nodedb-vector/src/hnsw/graph.rs

@@ -8,6 +8,11 @@ use crate::distance::distance;
 // Re-export shared params from nodedb-types.
 pub use nodedb_types::hnsw::HnswParams;
 
+/// Hard cap on the layer assigned to any node during insertion.
+/// Standard HNSW practice — prevents pathological RNG draws from inflating
+/// `max_layer` and slowing every subsequent search.
+pub const MAX_LAYER_CAP: usize = 16;
+
 /// Result of a k-NN search.
 #[derive(Debug, Clone)]
 pub struct SearchResult {
@@ -254,10 +259,15 @@ impl HnswIndex {
     }
 
     /// Assign a random layer using the exponential distribution.
+    ///
+    /// Capped at `MAX_LAYER_CAP` to prevent pathological RNG draws from
+    /// promoting the index's `max_layer` to hundreds or thousands, which
+    /// would make every search's Phase-1 greedy descent O(max_layer).
     pub(crate) fn random_layer(&mut self) -> usize {
         let ml = 1.0 / (self.params.m as f64).ln();
         let r = self.rng.next_f64().max(f64::MIN_POSITIVE);
-        (-r.ln() * ml).floor() as usize
+        let layer = (-r.ln() * ml).floor() as usize;
+        layer.min(MAX_LAYER_CAP)
     }
 
     /// Compute distance between a query vector and a stored node.
@@ -279,10 +289,22 @@ impl HnswIndex {
     }
 
     /// Compact the index by removing all tombstoned nodes.
+    ///
+    /// Returns the number of removed nodes. See `compact_with_map` for the
+    /// variant that also returns the old→new id remapping.
     pub fn compact(&mut self) -> usize {
+        self.compact_with_map().0
+    }
+
+    /// Compact and return both the removed count and the old→new id map.
+    ///
+    /// `id_map[old_local]` = new_local, or `u32::MAX` if the node was
+    /// tombstoned (removed).
+    pub fn compact_with_map(&mut self) -> (usize, Vec<u32>) {
         let tombstone_count = self.tombstone_count();
         if tombstone_count == 0 {
-            return 0;
+            let identity: Vec<u32> = (0..self.nodes.len() as u32).collect();
+            return (0, identity);
         }
         self.ensure_mutable_neighbors();
 
@@ -348,7 +370,7 @@ impl HnswIndex {
             .unwrap_or(0);
 
         self.nodes = new_nodes;
-        tombstone_count
+        (tombstone_count, id_map)
     }
 }
 

nodedb-vector/tests/hnsw_layer_cap.rs

@@ -0,0 +1,73 @@
+//! HNSW `random_layer` must be capped at a reasonable maximum.
+//!
+//! Spec: standard HNSW caps the assigned layer at ~16. The current
+//! `random_layer` implementation has no cap — with an unlucky xorshift
+//! draw (`r ≈ 2.2e-308`), `-ln(r) * (1/ln(m))` can return a layer in
+//! the hundreds or thousands. One outlier insert then promotes the
+//! index's `max_layer`, and every subsequent search's Phase-1 greedy
+//! descent iterates `(1..=max_layer).rev()` — converting constant-time
+//! descent into O(max_layer) per query.
+
+use nodedb_vector::DistanceMetric;
+use nodedb_vector::hnsw::{HnswIndex, HnswParams};
+
+/// Hard cap enforced by `HnswIndex::random_layer`. Standard HNSW uses ~16
+/// and the implementation clamps at `MAX_LAYER_CAP = 16`.
+const LAYER_CAP: usize = 16;
+
+#[test]
+fn random_layer_never_exceeds_cap_under_normal_inserts() {
+    let mut idx = HnswIndex::with_seed(
+        4,
+        HnswParams {
+            m: 16,
+            m0: 32,
+            ef_construction: 64,
+            metric: DistanceMetric::L2,
+        },
+        1,
+    );
+    for i in 0..5_000u32 {
+        let v = vec![
+            (i as f32).sin(),
+            (i as f32).cos(),
+            ((i * 3) as f32).sin(),
+            ((i * 7) as f32).cos(),
+        ];
+        idx.insert(v).unwrap();
+    }
+    assert!(
+        idx.max_layer() <= LAYER_CAP,
+        "max_layer grew to {} (cap = {LAYER_CAP}); one pathological random_layer \
+         draw promoted the index and will slow every subsequent search",
+        idx.max_layer()
+    );
+}
+
+#[test]
+fn random_layer_capped_with_adversarial_seed() {
+    // Seeds chosen to exercise xorshift states that produce very small
+    // `next_f64()` outputs early in the sequence. A correct implementation
+    // clamps the resulting layer regardless of the RNG draw.
+    for seed in [1u64, 2, 3, 7, 13, 42, 123, 9_999, 1_000_003] {
+        let mut idx = HnswIndex::with_seed(
+            2,
+            HnswParams {
+                m: 2, // small m amplifies -ln(r) * (1/ln(m))
+                m0: 4,
+                ef_construction: 32,
+                metric: DistanceMetric::L2,
+            },
+            seed,
+        );
+        for i in 0..2_000u32 {
+            idx.insert(vec![i as f32, 0.0]).unwrap();
+        }
+        assert!(
+            idx.max_layer() <= LAYER_CAP,
+            "seed={seed}: max_layer reached {} (cap = {LAYER_CAP}) — \
+             random_layer has no upper bound",
+            idx.max_layer()
+        );
+    }
+}