fix: add prev/next SemanticNeighbor edges between consecutive book-slice pages in HierarchyBuilder

Co-authored-by: devlux76 <86517969+devlux76@users.noreply.github.com>

Author: Copilot

hippocampus/HierarchyBuilder.ts

@@ -1,4 +1,4 @@
-import type { Book, Hash, MetadataStore, Shelf, Volume, VectorStore } from "../core/types";
+import type { Book, Hash, MetadataStore, SemanticNeighbor, Shelf, Volume, VectorStore } from "../core/types";
 import type { ModelProfile } from "../core/ModelProfile";
 import type { HotpathPolicy } from "../core/HotpathPolicy";
 import { hashText } from "../core/crypto/hash";
@@ -12,6 +12,11 @@ const PAGES_PER_BOOK = 8;
 const BOOKS_PER_VOLUME = 4;
 const VOLUMES_PER_SHELF = 4;
 
+// Max neighbors per page for the adjacency edges added by the hierarchy builder.
+// Adjacency edges represent document-order contiguity and bypass the cosine
+// cutoff used by FastNeighborInsert, so they must still be bounded by policy.
+const ADJACENCY_MAX_DEGREE = 16;
+
 export interface BuildHierarchyOptions {
   modelProfile: ModelProfile;
   vectorStore: VectorStore;
@@ -80,6 +85,36 @@ function chunkArray<T>(arr: T[], size: number): T[][] {
   return chunks;
 }
 
+/**
+ * Merge a candidate into a neighbor list, respecting maxDegree.
+ * If at capacity, evicts the neighbor with the lowest cosineSimilarity.
+ * Returns the updated list sorted by cosineSimilarity descending.
+ */
+function mergeAdjacentNeighbor(
+  existing: SemanticNeighbor[],
+  candidate: SemanticNeighbor,
+  maxDegree: number,
+): SemanticNeighbor[] {
+  const deduped = existing.filter((n) => n.neighborPageId !== candidate.neighborPageId);
+
+  if (deduped.length < maxDegree) {
+    deduped.push(candidate);
+  } else {
+    let weakestIdx = 0;
+    for (let i = 1; i < deduped.length; i++) {
+      if (deduped[i].cosineSimilarity < deduped[weakestIdx].cosineSimilarity) {
+        weakestIdx = i;
+      }
+    }
+    if (candidate.cosineSimilarity > deduped[weakestIdx].cosineSimilarity) {
+      deduped[weakestIdx] = candidate;
+    }
+  }
+
+  deduped.sort((a, b) => b.cosineSimilarity - a.cosineSimilarity);
+  return deduped;
+}
+
 export async function buildHierarchy(
   pageIds: Hash[],
   options: BuildHierarchyOptions,
@@ -99,6 +134,12 @@ export async function buildHierarchy(
   });
   const pageVectors = await vectorStore.readVectors(pageOffsets, dim);
 
+  // Build a Map<pageId, vector> for O(1) lookups throughout the hierarchy build.
+  const pageVectorMap = new Map<Hash, Float32Array>();
+  for (let i = 0; i < pageIds.length; i++) {
+    pageVectorMap.set(pageIds[i], pageVectors[i]);
+  }
+
   // -------------------------------------------------------------------------
   // Level 1: Pages → Books
   // -------------------------------------------------------------------------
@@ -110,8 +151,9 @@ export async function buildHierarchy(
     const bookId = await hashText(sortedChunk.join("|"));
 
     const chunkVectors = chunk.map((id) => {
-      const idx = pageIds.indexOf(id);
-      return pageVectors[idx];
+      const vec = pageVectorMap.get(id);
+      if (!vec) throw new Error(`Vector not found for page ${id}`);
+      return vec;
     });
 
     const medoidIdx = selectMedoidIndex(chunkVectors);
@@ -122,6 +164,32 @@ export async function buildHierarchy(
     books.push(book);
   }
 
+  // Add SemanticNeighbor edges between consecutive pages within each book slice.
+  // These document-order adjacency edges are always inserted regardless of cosine
+  // cutoff, because adjacent text chunks of the same source are always related.
+  for (const book of books) {
+    for (let i = 0; i < book.pageIds.length - 1; i++) {
+      const aId = book.pageIds[i];
+      const bId = book.pageIds[i + 1];
+      const aVec = pageVectorMap.get(aId);
+      const bVec = pageVectorMap.get(bId);
+      if (!aVec || !bVec) continue;
+
+      const sim = cosineSimilarity(aVec, bVec);
+      const dist = 1 - sim;
+      const forwardEdge: SemanticNeighbor = { neighborPageId: bId, cosineSimilarity: sim, distance: dist };
+      const reverseEdge: SemanticNeighbor = { neighborPageId: aId, cosineSimilarity: sim, distance: dist };
+
+      // Forward: a → b
+      const existingA = await metadataStore.getSemanticNeighbors(aId);
+      await metadataStore.putSemanticNeighbors(aId, mergeAdjacentNeighbor(existingA, forwardEdge, ADJACENCY_MAX_DEGREE));
+
+      // Reverse: b → a
+      const existingB = await metadataStore.getSemanticNeighbors(bId);
+      await metadataStore.putSemanticNeighbors(bId, mergeAdjacentNeighbor(existingB, reverseEdge, ADJACENCY_MAX_DEGREE));
+    }
+  }
+
   await runPromotionSweep(books.map((b) => b.bookId), metadataStore, policy);
 
   // -------------------------------------------------------------------------
@@ -135,8 +203,9 @@ export async function buildHierarchy(
     const volumeId = await hashText(sortedBookIds.join("|"));
 
     const medoidVectors = bookChunk.map((b) => {
-      const idx = pageIds.indexOf(b.medoidPageId);
-      return pageVectors[idx];
+      const vec = pageVectorMap.get(b.medoidPageId);
+      if (!vec) throw new Error(`Vector not found for medoid page ${b.medoidPageId}`);
+      return vec;
     });
 
     const centroid = computeCentroid(medoidVectors);

tests/hippocampus/FastNeighborInsert.test.ts

@@ -6,7 +6,8 @@ import { MemoryVectorStore } from "../../storage/MemoryVectorStore";
 import { DeterministicDummyEmbeddingBackend } from "../../embeddings/DeterministicDummyEmbeddingBackend";
 import { EmbeddingRunner } from "../../embeddings/EmbeddingRunner";
 import { generateKeyPair } from "../../core/crypto/sign";
-import { ingestText } from "../../hippocampus/Ingest";
+import { buildPage } from "../../hippocampus/PageBuilder";
+import { chunkText } from "../../hippocampus/Chunker";
 import { insertSemanticNeighbors } from "../../hippocampus/FastNeighborInsert";
 import type { ModelProfile } from "../../core/ModelProfile";
 
@@ -24,6 +25,11 @@ const PROFILE: ModelProfile = {
   source: "metadata",
 };
 
+/**
+ * Builds `pageCount` pages directly without calling ingestText/buildHierarchy,
+ * so the SemanticNeighbor graph starts empty. This keeps FastNeighborInsert
+ * tests fully isolated from HierarchyBuilder's adjacency-edge insertion.
+ */
 async function makeFixture(pageCount: number) {
   const metadataStore = await IndexedDbMetadataStore.open(freshDbName());
   const vectorStore = new MemoryVectorStore();
@@ -40,16 +46,31 @@ async function makeFixture(pageCount: number) {
 
   const words = Array.from({ length: pageCount * 4 }, (_, i) => `word${i}`);
   const text = words.join(" ");
+  const chunks = chunkText(text, PROFILE);
+  const useChunks = chunks.slice(0, pageCount);
+  const embeddings = await runner.embed(useChunks);
+
+  const createdAt = new Date().toISOString();
+  const pageIds: string[] = [];
+
+  for (let i = 0; i < useChunks.length; i++) {
+    const embedding = embeddings[i];
+    const offset = await vectorStore.appendVector(embedding);
+    const page = await buildPage({
+      content: useChunks[i],
+      embedding,
+      embeddingOffset: offset,
+      embeddingDim: PROFILE.embeddingDimension,
+      creatorPubKey: keyPair.publicKey,
+      signingKey: keyPair.signingKey,
+      createdAt,
+    });
+    await metadataStore.putPage(page);
+    await metadataStore.putPageActivity({ pageId: page.pageId, queryHitCount: 0, lastQueryAt: createdAt });
+    pageIds.push(page.pageId);
+  }
 
-  const result = await ingestText(text, {
-    modelProfile: PROFILE,
-    embeddingRunner: runner,
-    vectorStore,
-    metadataStore,
-    keyPair,
-  });
-
-  return { metadataStore, vectorStore, pageIds: result.pages.map((p) => p.pageId) };
+  return { metadataStore, vectorStore, pageIds };
 }
 
 describe("FastNeighborInsert", () => {

tests/hippocampus/HierarchyBuilder.test.ts

@@ -286,4 +286,39 @@ describe("HierarchyBuilder", () => {
     expect(result.volumes!.length).toBeGreaterThanOrEqual(1);
     expect(result.shelves!.length).toBeGreaterThanOrEqual(1);
   });
+
+  it("adds SemanticNeighbor edges between consecutive pages within each book slice", async () => {
+    const { metadataStore, vectorStore, pageIds } = await makeFixture(4);
+
+    const { books } = await buildHierarchy(pageIds, {
+      modelProfile: PROFILE,
+      vectorStore,
+      metadataStore,
+    });
+
+    // For each book with at least 2 pages, every consecutive pair should have
+    // a SemanticNeighbor edge in both directions.
+    for (const book of books) {
+      for (let i = 0; i < book.pageIds.length - 1; i++) {
+        const aId = book.pageIds[i];
+        const bId = book.pageIds[i + 1];
+
+        // Forward: a → b
+        const aNeighbors = await metadataStore.getSemanticNeighbors(aId);
+        const aHasB = aNeighbors.some((n) => n.neighborPageId === bId);
+        expect(aHasB).toBe(true);
+
+        // Reverse: b → a
+        const bNeighbors = await metadataStore.getSemanticNeighbors(bId);
+        const bHasA = bNeighbors.some((n) => n.neighborPageId === aId);
+        expect(bHasA).toBe(true);
+
+        // Edge data should be structurally valid.
+        const edge = aNeighbors.find((n) => n.neighborPageId === bId)!;
+        expect(edge.cosineSimilarity).toBeGreaterThanOrEqual(-1);
+        expect(edge.cosineSimilarity).toBeLessThanOrEqual(1);
+        expect(edge.distance).toBeCloseTo(1 - edge.cosineSimilarity, 5);
+      }
+    }
+  });
 });