diff --git a/KNOWN-ISSUES.adoc b/KNOWN-ISSUES.adoc index 8dbc03c..4957801 100644 --- a/KNOWN-ISSUES.adoc +++ b/KNOWN-ISSUES.adoc @@ -107,13 +107,13 @@ Added `rescript.json` for build configuration. **Impact:** None (resolved). Documented here for audit trail completeness. -=== 9. HNSW Implementation Is Real +=== 9. HNSW Implementation Is Real — and now the default -**Location:** `rust-core/verisim-vector/src/hnsw.rs` +**Location:** `rust-core/verisim-vector/src/hnsw.rs`, `rust-core/verisim-api/src/lib.rs` -**Current state:** The HNSW (Hierarchical Navigable Small World) implementation in `verisim-vector` is a genuine, functional implementation at approximately 670 lines of Rust. A previous audit incorrectly claimed this was brute-force search. This is **not** an issue -- it is a correction of a previous mischaracterization. +**Current state:** The HNSW (Hierarchical Navigable Small World) implementation in `verisim-vector` is a genuine, functional implementation at approximately 670 lines of Rust (real Malkov–Yashunin layered graph, configurable `M`/`ef_construction`/`ef_search`, cosine/euclidean/dot metrics). -**Impact:** None (this is a positive clarification). The vector modality store uses a real approximate nearest-neighbor algorithm, not a naive linear scan. +**Resolved (wiring):** 2026-06-13. Until now this module was *built but orphaned* — the API's `ConcreteOctadStore` used `BruteForceVectorStore` (exact O(n) linear scan), so the "HNSW similarity search" claim was aspirational at the product boundary. The in-memory `ConcreteOctadStore` now uses `HnswVectorStore` as its vector backend, so the default build performs real approximate nearest-neighbor search. (The persistent/redb vector backend remains brute-force over an mmap'd store — a separate follow-on.) **Note:** This entry exists to prevent future audits from repeating the same incorrect claim. @@ -267,3 +267,11 @@ Added `rescript.json` for build configuration. * `temporal_consistency_drift` works at 1-second timestamp resolution, so two writes within the same second register as a (mild) conflict signal. **Original issue:** The product's core claim — drift detection — ran on no real data anywhere in the system. + +=== 27. Search Scores Were Fabricated From Rank — ✅ RESOLVED + +**Location:** `rust-core/verisim-api/src/lib.rs`, `rust-core/verisim-octad/src/{lib,store}.rs` + +**Resolved:** 2026-06-13. The text, vector, and similar-query search endpoints all reported `score: 1.0 - (i as f32 * 0.1)` — a synthetic sequence derived from result position, discarding the real relevance scores the modality stores already compute (Tantivy BM25 for documents, cosine similarity for vectors). The octad store now exposes `search_text_scored` / `search_similar_scored` (returning `Vec<(Octad, f32)>`); the three handlers surface those real scores. Test `test_vector_search_returns_real_cosine_scores` pins the distinction (the second hit reports its true ~0.994 cosine, not synthetic 0.9). + +**Original issue:** API search responses carried fabricated scores, so any client ranking or thresholding on `score` was meaningless. diff --git a/rust-core/verisim-api/src/lib.rs b/rust-core/verisim-api/src/lib.rs index 1064c4c..ec06ddb 100644 --- a/rust-core/verisim-api/src/lib.rs +++ b/rust-core/verisim-api/src/lib.rs @@ -77,9 +77,9 @@ use verisim_temporal::RedbVersionStore; use verisim_tensor::InMemoryTensorStore; #[cfg(feature = "persistent")] use verisim_tensor::RedbTensorStore; -#[cfg(not(feature = "persistent"))] -use verisim_vector::BruteForceVectorStore; use verisim_vector::DistanceMetric; +#[cfg(not(feature = "persistent"))] +use verisim_vector::HnswVectorStore; #[cfg(feature = "persistent")] use verisim_vector::RedbVectorStore; @@ -91,7 +91,7 @@ use verisim_vector::RedbVectorStore; #[cfg(not(feature = "persistent"))] pub type ConcreteOctadStore = InMemoryOctadStore< SimpleGraphStore, - BruteForceVectorStore, + HnswVectorStore, TantivyDocumentStore, InMemoryTensorStore, InMemorySemanticStore, @@ -580,7 +580,7 @@ impl AppState { .map_err(|e| ApiError::Internal(e.to_string()))?, ); #[cfg(not(feature = "persistent"))] - let vector = Arc::new(BruteForceVectorStore::new( + let vector = Arc::new(HnswVectorStore::with_defaults( config.vector_dimension, DistanceMetric::Cosine, )); @@ -1153,18 +1153,17 @@ async fn text_search_handler( }; let limit = validate_limit(query.limit.unwrap_or(10)); - let octads = state + let scored = state .octad_store - .search_text(&q, limit) + .search_text_scored(&q, limit) .await .map_err(|e| ApiError::Internal(e.to_string()))?; - let results: Vec = octads + let results: Vec = scored .iter() - .enumerate() - .map(|(i, h)| SearchResultResponse { + .map(|(h, score)| SearchResultResponse { id: h.id.to_string(), - score: 1.0 - (i as f32 * 0.1), // Approximate score based on ranking + score: *score, // Tantivy BM25 relevance score title: h.document.as_ref().map(|d| d.title.clone()), }) .collect(); @@ -1189,18 +1188,17 @@ async fn vector_search_handler( } validate_vector(&request.vector)?; - let octads = state + let scored = state .octad_store - .search_similar(&request.vector, k) + .search_similar_scored(&request.vector, k) .await .map_err(|e| ApiError::Internal(e.to_string()))?; - let results: Vec = octads + let results: Vec = scored .iter() - .enumerate() - .map(|(i, h)| SearchResultResponse { + .map(|(h, score)| SearchResultResponse { id: h.id.to_string(), - score: 1.0 - (i as f32 * 0.1), // Approximate score based on ranking + score: *score, // cosine similarity from the vector index title: h.document.as_ref().map(|d| d.title.clone()), }) .collect(); @@ -1594,25 +1592,24 @@ async fn similar_queries_handler( validate_vector(&request.vector)?; // Search for similar octads (which includes query-octads) - let octads = state + let scored = state .octad_store - .search_similar(&request.vector, k) + .search_similar_scored(&request.vector, k) .await .map_err(|e| ApiError::Internal(e.to_string()))?; // Filter to only query octads (those with "vql_query" type in document fields) - let results: Vec = octads + let results: Vec = scored .iter() - .filter(|h| { + .filter(|(h, _score)| { h.document .as_ref() .map(|d| d.title.starts_with("VQL Query:")) .unwrap_or(false) }) - .enumerate() - .map(|(i, h)| SearchResultResponse { + .map(|(h, score)| SearchResultResponse { id: h.id.to_string(), - score: 1.0 - (i as f32 * 0.1), + score: *score, // cosine similarity from the vector index title: h.document.as_ref().map(|d| d.title.clone()), }) .collect(); @@ -2811,6 +2808,76 @@ mod tests { assert_eq!(response.status(), StatusCode::OK); } + /// Vector search must surface the index's real cosine scores, not a + /// synthetic rank sequence (the old `1.0 - 0.1*i`). The discriminator: + /// `[0.9, 0.1, 0.0]` is ~0.994 cosine to `[1,0,0]`, whereas the old + /// scheme would have reported the second hit as exactly 0.9. + #[tokio::test] + async fn test_vector_search_returns_real_cosine_scores() { + let state = create_test_state().await; + let app = build_router(state); + + let make = |embedding: Vec, title: &str| OctadRequest { + title: Some(title.to_string()), + body: Some("body".to_string()), + embedding: Some(embedding), + types: None, + relationships: None, + tensor: None, + metadata: None, + provenance: None, + spatial: None, + }; + + post_octad(&app, &make(vec![1.0, 0.0, 0.0], "aligned")).await; + post_octad(&app, &make(vec![0.9, 0.1, 0.0], "near")).await; + post_octad(&app, &make(vec![0.0, 1.0, 0.0], "orthogonal")).await; + + let request = VectorSearchRequest { + vector: vec![1.0, 0.0, 0.0], + k: Some(3), + }; + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/search/vector") + .header("content-type", "application/json") + .body(Body::from(serde_json::to_string(&request).unwrap())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(); + let results: Vec = serde_json::from_slice(&body).unwrap(); + assert!(results.len() >= 2, "expected at least two hits"); + + // Top hit is the identical-direction vector: cosine ~1.0. + assert!( + results[0].score > 0.99, + "top cosine score should be ~1.0, got {}", + results[0].score + ); + // Second hit's real cosine is ~0.994; the old synthetic scheme would + // have reported exactly 0.9. This is the load-bearing assertion. + assert!( + results[1].score > 0.95, + "second hit must carry its real cosine (~0.994), not synthetic 0.9; got {}", + results[1].score + ); + // Scores are non-increasing in relevance order. + for pair in results.windows(2) { + assert!( + pair[0].score >= pair[1].score, + "scores must be in descending relevance order" + ); + } + } + #[tokio::test] async fn test_drift_status() { let state = create_test_state().await; diff --git a/rust-core/verisim-octad/src/lib.rs b/rust-core/verisim-octad/src/lib.rs index f70e938..962a0e2 100644 --- a/rust-core/verisim-octad/src/lib.rs +++ b/rust-core/verisim-octad/src/lib.rs @@ -324,9 +324,25 @@ pub trait OctadStore: Send + Sync { /// Search by vector similarity async fn search_similar(&self, embedding: &[f32], k: usize) -> Result, OctadError>; + /// Search by vector similarity, returning each match with its modality + /// score (cosine similarity for the default metric; higher is closer). + async fn search_similar_scored( + &self, + embedding: &[f32], + k: usize, + ) -> Result, OctadError>; + /// Search by document text async fn search_text(&self, query: &str, limit: usize) -> Result, OctadError>; + /// Search by document text, returning each match with its relevance + /// score (Tantivy BM25; higher is more relevant). + async fn search_text_scored( + &self, + query: &str, + limit: usize, + ) -> Result, OctadError>; + /// Query by graph relationship async fn query_related(&self, id: &OctadId, predicate: &str) -> Result, OctadError>; diff --git a/rust-core/verisim-octad/src/store.rs b/rust-core/verisim-octad/src/store.rs index e2c9c88..7b99843 100644 --- a/rust-core/verisim-octad/src/store.rs +++ b/rust-core/verisim-octad/src/store.rs @@ -1385,6 +1385,19 @@ where } async fn search_similar(&self, embedding: &[f32], k: usize) -> Result, OctadError> { + Ok(self + .search_similar_scored(embedding, k) + .await? + .into_iter() + .map(|(octad, _score)| octad) + .collect()) + } + + async fn search_similar_scored( + &self, + embedding: &[f32], + k: usize, + ) -> Result, OctadError> { let results = self.vector .search(embedding, k) @@ -1394,10 +1407,12 @@ where message: e.to_string(), })?; + // Preserve the vector store's relevance order and carry each score + // through to the caller (the API surfaces it; no fabricated ranking). let mut octads = Vec::new(); for result in results { if let Some(octad) = self.load_octad(&OctadId::new(&result.id)).await? { - octads.push(octad); + octads.push((octad, result.score)); } } @@ -1405,6 +1420,19 @@ where } async fn search_text(&self, query: &str, limit: usize) -> Result, OctadError> { + Ok(self + .search_text_scored(query, limit) + .await? + .into_iter() + .map(|(octad, _score)| octad) + .collect()) + } + + async fn search_text_scored( + &self, + query: &str, + limit: usize, + ) -> Result, OctadError> { let results = self.document .search(query, limit) @@ -1417,7 +1445,7 @@ where let mut octads = Vec::new(); for result in results { if let Some(octad) = self.load_octad(&OctadId::new(&result.id)).await? { - octads.push(octad); + octads.push((octad, result.score)); } }