feat(search): real HNSW default + real relevance scores end-to-end

KNOWN-ISSUES.adoc

@@ -107,13 +107,13 @@ Added `rescript.json` for build configuration.
 
 **Impact:** None (resolved). Documented here for audit trail completeness.
 
-=== 9. HNSW Implementation Is Real
+=== 9. HNSW Implementation Is Real — and now the default
 
-**Location:** `rust-core/verisim-vector/src/hnsw.rs`
+**Location:** `rust-core/verisim-vector/src/hnsw.rs`, `rust-core/verisim-api/src/lib.rs`
 
-**Current state:** The HNSW (Hierarchical Navigable Small World) implementation in `verisim-vector` is a genuine, functional implementation at approximately 670 lines of Rust. A previous audit incorrectly claimed this was brute-force search. This is **not** an issue -- it is a correction of a previous mischaracterization.
+**Current state:** The HNSW (Hierarchical Navigable Small World) implementation in `verisim-vector` is a genuine, functional implementation at approximately 670 lines of Rust (real Malkov–Yashunin layered graph, configurable `M`/`ef_construction`/`ef_search`, cosine/euclidean/dot metrics).
 
-**Impact:** None (this is a positive clarification). The vector modality store uses a real approximate nearest-neighbor algorithm, not a naive linear scan.
+**Resolved (wiring):** 2026-06-13. Until now this module was *built but orphaned* — the API's `ConcreteOctadStore` used `BruteForceVectorStore` (exact O(n) linear scan), so the "HNSW similarity search" claim was aspirational at the product boundary. The in-memory `ConcreteOctadStore` now uses `HnswVectorStore` as its vector backend, so the default build performs real approximate nearest-neighbor search. (The persistent/redb vector backend remains brute-force over an mmap'd store — a separate follow-on.)
 
 **Note:** This entry exists to prevent future audits from repeating the same incorrect claim.
 
@@ -267,3 +267,11 @@ Added `rescript.json` for build configuration.
 * `temporal_consistency_drift` works at 1-second timestamp resolution, so two writes within the same second register as a (mild) conflict signal.
 
 **Original issue:** The product's core claim — drift detection — ran on no real data anywhere in the system.
+
+=== 27. Search Scores Were Fabricated From Rank — ✅ RESOLVED
+
+**Location:** `rust-core/verisim-api/src/lib.rs`, `rust-core/verisim-octad/src/{lib,store}.rs`
+
+**Resolved:** 2026-06-13. The text, vector, and similar-query search endpoints all reported `score: 1.0 - (i as f32 * 0.1)` — a synthetic sequence derived from result position, discarding the real relevance scores the modality stores already compute (Tantivy BM25 for documents, cosine similarity for vectors). The octad store now exposes `search_text_scored` / `search_similar_scored` (returning `Vec<(Octad, f32)>`); the three handlers surface those real scores. Test `test_vector_search_returns_real_cosine_scores` pins the distinction (the second hit reports its true ~0.994 cosine, not synthetic 0.9).
+
+**Original issue:** API search responses carried fabricated scores, so any client ranking or thresholding on `score` was meaningless.

rust-core/verisim-api/src/lib.rs

@@ -77,9 +77,9 @@ use verisim_temporal::RedbVersionStore;
 use verisim_tensor::InMemoryTensorStore;
 #[cfg(feature = "persistent")]
 use verisim_tensor::RedbTensorStore;
-#[cfg(not(feature = "persistent"))]
-use verisim_vector::BruteForceVectorStore;
 use verisim_vector::DistanceMetric;
+#[cfg(not(feature = "persistent"))]
+use verisim_vector::HnswVectorStore;
 #[cfg(feature = "persistent")]
 use verisim_vector::RedbVectorStore;
 
@@ -91,7 +91,7 @@ use verisim_vector::RedbVectorStore;
 #[cfg(not(feature = "persistent"))]
 pub type ConcreteOctadStore = InMemoryOctadStore<
     SimpleGraphStore,
-    BruteForceVectorStore,
+    HnswVectorStore,
     TantivyDocumentStore,
     InMemoryTensorStore,
     InMemorySemanticStore,
@@ -580,7 +580,7 @@ impl AppState {
             .map_err(|e| ApiError::Internal(e.to_string()))?,
         );
         #[cfg(not(feature = "persistent"))]
-        let vector = Arc::new(BruteForceVectorStore::new(
+        let vector = Arc::new(HnswVectorStore::with_defaults(
             config.vector_dimension,
             DistanceMetric::Cosine,
         ));
@@ -1153,18 +1153,17 @@ async fn text_search_handler(
     };
     let limit = validate_limit(query.limit.unwrap_or(10));
 
-    let octads = state
+    let scored = state
         .octad_store
-        .search_text(&q, limit)
+        .search_text_scored(&q, limit)
         .await
         .map_err(|e| ApiError::Internal(e.to_string()))?;
 
-    let results: Vec<SearchResultResponse> = octads
+    let results: Vec<SearchResultResponse> = scored
         .iter()
-        .enumerate()
-        .map(|(i, h)| SearchResultResponse {
+        .map(|(h, score)| SearchResultResponse {
             id: h.id.to_string(),
-            score: 1.0 - (i as f32 * 0.1), // Approximate score based on ranking
+            score: *score, // Tantivy BM25 relevance score
             title: h.document.as_ref().map(|d| d.title.clone()),
         })
         .collect();
@@ -1189,18 +1188,17 @@ async fn vector_search_handler(
     }
     validate_vector(&request.vector)?;
 
-    let octads = state
+    let scored = state
         .octad_store
-        .search_similar(&request.vector, k)
+        .search_similar_scored(&request.vector, k)
         .await
         .map_err(|e| ApiError::Internal(e.to_string()))?;
 
-    let results: Vec<SearchResultResponse> = octads
+    let results: Vec<SearchResultResponse> = scored
         .iter()
-        .enumerate()
-        .map(|(i, h)| SearchResultResponse {
+        .map(|(h, score)| SearchResultResponse {
             id: h.id.to_string(),
-            score: 1.0 - (i as f32 * 0.1), // Approximate score based on ranking
+            score: *score, // cosine similarity from the vector index
             title: h.document.as_ref().map(|d| d.title.clone()),
         })
         .collect();
@@ -1594,25 +1592,24 @@ async fn similar_queries_handler(
     validate_vector(&request.vector)?;
 
     // Search for similar octads (which includes query-octads)
-    let octads = state
+    let scored = state
         .octad_store
-        .search_similar(&request.vector, k)
+        .search_similar_scored(&request.vector, k)
         .await
         .map_err(|e| ApiError::Internal(e.to_string()))?;
 
     // Filter to only query octads (those with "vql_query" type in document fields)
-    let results: Vec<SearchResultResponse> = octads
+    let results: Vec<SearchResultResponse> = scored
         .iter()
-        .filter(|h| {
+        .filter(|(h, _score)| {
             h.document
                 .as_ref()
                 .map(|d| d.title.starts_with("VQL Query:"))
                 .unwrap_or(false)
         })
-        .enumerate()
-        .map(|(i, h)| SearchResultResponse {
+        .map(|(h, score)| SearchResultResponse {
             id: h.id.to_string(),
-            score: 1.0 - (i as f32 * 0.1),
+            score: *score, // cosine similarity from the vector index
             title: h.document.as_ref().map(|d| d.title.clone()),
         })
         .collect();
@@ -2811,6 +2808,76 @@ mod tests {
         assert_eq!(response.status(), StatusCode::OK);
     }
 
+    /// Vector search must surface the index's real cosine scores, not a
+    /// synthetic rank sequence (the old `1.0 - 0.1*i`). The discriminator:
+    /// `[0.9, 0.1, 0.0]` is ~0.994 cosine to `[1,0,0]`, whereas the old
+    /// scheme would have reported the second hit as exactly 0.9.
+    #[tokio::test]
+    async fn test_vector_search_returns_real_cosine_scores() {
+        let state = create_test_state().await;
+        let app = build_router(state);
+
+        let make = |embedding: Vec<f32>, title: &str| OctadRequest {
+            title: Some(title.to_string()),
+            body: Some("body".to_string()),
+            embedding: Some(embedding),
+            types: None,
+            relationships: None,
+            tensor: None,
+            metadata: None,
+            provenance: None,
+            spatial: None,
+        };
+
+        post_octad(&app, &make(vec![1.0, 0.0, 0.0], "aligned")).await;
+        post_octad(&app, &make(vec![0.9, 0.1, 0.0], "near")).await;
+        post_octad(&app, &make(vec![0.0, 1.0, 0.0], "orthogonal")).await;
+
+        let request = VectorSearchRequest {
+            vector: vec![1.0, 0.0, 0.0],
+            k: Some(3),
+        };
+        let response = app
+            .oneshot(
+                Request::builder()
+                    .method("POST")
+                    .uri("/search/vector")
+                    .header("content-type", "application/json")
+                    .body(Body::from(serde_json::to_string(&request).unwrap()))
+                    .unwrap(),
+            )
+            .await
+            .unwrap();
+        assert_eq!(response.status(), StatusCode::OK);
+
+        let body = axum::body::to_bytes(response.into_body(), 1024 * 1024)
+            .await
+            .unwrap();
+        let results: Vec<SearchResultResponse> = serde_json::from_slice(&body).unwrap();
+        assert!(results.len() >= 2, "expected at least two hits");
+
+        // Top hit is the identical-direction vector: cosine ~1.0.
+        assert!(
+            results[0].score > 0.99,
+            "top cosine score should be ~1.0, got {}",
+            results[0].score
+        );
+        // Second hit's real cosine is ~0.994; the old synthetic scheme would
+        // have reported exactly 0.9. This is the load-bearing assertion.
+        assert!(
+            results[1].score > 0.95,
+            "second hit must carry its real cosine (~0.994), not synthetic 0.9; got {}",
+            results[1].score
+        );
+        // Scores are non-increasing in relevance order.
+        for pair in results.windows(2) {
+            assert!(
+                pair[0].score >= pair[1].score,
+                "scores must be in descending relevance order"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn test_drift_status() {
         let state = create_test_state().await;

rust-core/verisim-octad/src/lib.rs

@@ -324,9 +324,25 @@ pub trait OctadStore: Send + Sync {
     /// Search by vector similarity
     async fn search_similar(&self, embedding: &[f32], k: usize) -> Result<Vec<Octad>, OctadError>;
 
+    /// Search by vector similarity, returning each match with its modality
+    /// score (cosine similarity for the default metric; higher is closer).
+    async fn search_similar_scored(
+        &self,
+        embedding: &[f32],
+        k: usize,
+    ) -> Result<Vec<(Octad, f32)>, OctadError>;
+
     /// Search by document text
     async fn search_text(&self, query: &str, limit: usize) -> Result<Vec<Octad>, OctadError>;
 
+    /// Search by document text, returning each match with its relevance
+    /// score (Tantivy BM25; higher is more relevant).
+    async fn search_text_scored(
+        &self,
+        query: &str,
+        limit: usize,
+    ) -> Result<Vec<(Octad, f32)>, OctadError>;
+
     /// Query by graph relationship
     async fn query_related(&self, id: &OctadId, predicate: &str) -> Result<Vec<Octad>, OctadError>;
 

rust-core/verisim-octad/src/store.rs

@@ -1385,6 +1385,19 @@ where
     }
 
     async fn search_similar(&self, embedding: &[f32], k: usize) -> Result<Vec<Octad>, OctadError> {
+        Ok(self
+            .search_similar_scored(embedding, k)
+            .await?
+            .into_iter()
+            .map(|(octad, _score)| octad)
+            .collect())
+    }
+
+    async fn search_similar_scored(
+        &self,
+        embedding: &[f32],
+        k: usize,
+    ) -> Result<Vec<(Octad, f32)>, OctadError> {
         let results =
             self.vector
                 .search(embedding, k)
@@ -1394,17 +1407,32 @@ where
                     message: e.to_string(),
                 })?;
 
+        // Preserve the vector store's relevance order and carry each score
+        // through to the caller (the API surfaces it; no fabricated ranking).
         let mut octads = Vec::new();
         for result in results {
             if let Some(octad) = self.load_octad(&OctadId::new(&result.id)).await? {
-                octads.push(octad);
+                octads.push((octad, result.score));
             }
         }
 
         Ok(octads)
     }
 
     async fn search_text(&self, query: &str, limit: usize) -> Result<Vec<Octad>, OctadError> {
+        Ok(self
+            .search_text_scored(query, limit)
+            .await?
+            .into_iter()
+            .map(|(octad, _score)| octad)
+            .collect())
+    }
+
+    async fn search_text_scored(
+        &self,
+        query: &str,
+        limit: usize,
+    ) -> Result<Vec<(Octad, f32)>, OctadError> {
         let results =
             self.document
                 .search(query, limit)
@@ -1417,7 +1445,7 @@ where
         let mut octads = Vec::new();
         for result in results {
             if let Some(octad) = self.load_octad(&OctadId::new(&result.id)).await? {
-                octads.push(octad);
+                octads.push((octad, result.score));
             }
         }