fix(embeddings): fail loudly instead of storing zero vectors when embedding fails

CHANGELOG.md

@@ -9,6 +9,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+- Ingest and retrieval no longer degrade silently when the embedding provider fails (for
+  example on an exhausted API quota). A zero vector is never stored or queried: `add_documents`
+  rejects a zero-norm vector produced for a non-empty chunk, `search` rejects a zero-norm query,
+  and `add_papers` propagates the failure instead of reporting a successful ingest of zero
+  chunks. Previously a poisoned collection answered every query with the same passages at a
+  constant score of 0.5 while the API reported `success: true`.
+- `add_documents` rejects a provider that returns fewer vectors than texts, which otherwise
+  assigned each embedding to the wrong chunk. `screening.py` gained the same guard.
+
+### Added
+- `GET /api/kb/{name}/stats` reports an `embedding_health` block (`probed_chunks`,
+  `zero_vector_chunks`, `degraded`) so a poisoned knowledge base is visible without reading
+  Chroma by hand.
+
 ### Changed
 - **BREAKING:** the `indicia` and `adapters` optional extras are removed. They required the
   private, unpublished `indicium` stack, and `uv lock` resolves every extra whether or not it

src/perspicacite/llm/embeddings.py

@@ -1,5 +1,6 @@
 """Embedding providers for vector search."""
 
+import math
 from pathlib import Path
 from typing import Any, Protocol
 
@@ -8,8 +9,28 @@
 logger = get_logger("perspicacite.llm.embeddings")
 
 
+class EmbeddingFailedError(RuntimeError):
+    """An embedding could not be produced, so no vector may be stored or queried.
+
+    Raised instead of substituting a zero vector. A zero vector is not a
+    neutral placeholder: in cosine space it sits at distance 1.0 from every
+    other vector, which silently turns retrieval into a constant score over an
+    arbitrary ordering rather than an error.
+    """
+
+
+def is_zero_vector(vector: list[float], tolerance: float = 1e-12) -> bool:
+    """Return True when the vector's L2 norm is indistinguishable from zero."""
+    return math.sqrt(sum(component * component for component in vector)) <= tolerance
+
+
 class EmbeddingProvider(Protocol):
-    """Protocol for embedding providers."""
+    """Protocol for embedding providers.
+
+    Contract for implementers: raise on failure, never substitute a zero
+    vector for a text that has content. A zero vector is only ever a valid
+    answer for input that is itself empty or whitespace.
+    """
 
     async def embed(self, texts: list[str]) -> list[list[float]]:
         """Embed a list of texts (document mode)."""

src/perspicacite/pipeline/search_to_kb.py

@@ -37,6 +37,7 @@
 from dataclasses import dataclass, field
 from typing import Any
 
+from perspicacite.llm.embeddings import EmbeddingFailedError
 from perspicacite.logging import get_logger
 
 logger = get_logger("perspicacite.pipeline.search_to_kb")
@@ -547,6 +548,9 @@ async def ingest_dois_into_kb(
         cookies_path = pdf_config.cookies_path
 
     papers_to_add: list[Paper] = []
+    # DOIs marked "added" on retrieval but not yet embedded. The checkpoint never
+    # re-offers an "added" id, so the mark must be undone if the batch never lands.
+    pending_dois: list[str] = []
     skipped: list[dict] = []
     failed: list[dict[str, str]] = []
     dl: dict[str, int] = {"attempted": 0, "success": 0, "failed": 0}
@@ -632,6 +636,7 @@ async def ingest_dois_into_kb(
             # Mark as added immediately on successful retrieval (Wave 3.3).
             ck_state.record(doi, "added")
             ckpt.save(ck_state)
+            pending_dois.append(doi)
 
     added_chunks = 0
     if papers_to_add:
@@ -641,7 +646,17 @@ async def ingest_dois_into_kb(
         )
         dkb.collection_name = collection_name
         dkb._initialized = True
-        added_chunks = await dkb.add_papers(papers_to_add, include_full_text=True)
+        try:
+            added_chunks = await dkb.add_papers(papers_to_add, include_full_text=True)
+        except EmbeddingFailedError:
+            # These DOIs were marked "added" the moment they were retrieved, before
+            # the batch was embedded. remaining_ids() never re-offers an "added" id,
+            # so leaving the mark would make a resume skip them for ever. Chroma
+            # ignores duplicate ids, so re-ingesting any that did land is harmless.
+            for pending in pending_dois:
+                ck_state.processed.pop(pending, None)
+            ckpt.save(ck_state)
+            raise
         kb_meta.paper_count = (kb_meta.paper_count or 0) + len(papers_to_add)
         kb_meta.chunk_count = (kb_meta.chunk_count or 0) + added_chunks
         await app_state.session_store.save_kb_metadata(kb_meta)

src/perspicacite/rag/dynamic_kb.py

@@ -9,6 +9,7 @@
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any
 
+from perspicacite.llm.embeddings import EmbeddingFailedError
 from perspicacite.logging import get_logger
 from perspicacite.rag.paper_metadata_codec import decode_paper_metadata_json
 from perspicacite.rag.query_scope import PaperScopeResult, merge_scope_with_candidates
@@ -116,6 +117,12 @@ async def add_papers(
                 total_added += docs_added
                 self._paper_ids.add(paper.id)
 
+            except EmbeddingFailedError:
+                # The embedder is down (bad key, exhausted quota). Every
+                # remaining paper would fail the same way, and swallowing it
+                # would report a successful ingest of zero usable chunks.
+                logger.error("add_paper_embedding_failed", paper_id=paper.id)
+                raise
             except Exception as e:
                 logger.error("add_paper_error", paper_id=paper.id, error=str(e))
 

src/perspicacite/rag/modes/advanced.py

@@ -15,6 +15,7 @@
 from typing import Any
 
 from perspicacite.config.schema import MultimodalMode
+from perspicacite.llm.embeddings import EmbeddingFailedError
 from perspicacite.logging import get_logger
 from perspicacite.models.kb import chroma_collection_name_for_kb
 from perspicacite.models.rag import RAGMode, RAGRequest, RAGResponse, SourceReference, StreamEvent
@@ -1050,6 +1051,8 @@ async def _wrrf_retrieval(
                             query_embedding=query_embedding[0],
                             top_k=self.initial_docs,
                         )
+                    except EmbeddingFailedError:
+                        raise  # a degenerate query fails against every collection
                     except Exception as e:
                         logger.warning(
                             "advanced_wrrf_fanout_search_failed",

src/perspicacite/rag/modes/deep_research.py

@@ -21,6 +21,7 @@
 from typing import Any
 
 from perspicacite.config.schema import MultimodalMode
+from perspicacite.llm.embeddings import EmbeddingFailedError
 from perspicacite.logging import get_logger
 from perspicacite.models.kb import chroma_collection_name_for_kb
 from perspicacite.models.rag import RAGMode, RAGRequest, RAGResponse, SourceReference, StreamEvent
@@ -1411,6 +1412,8 @@ async def _wrrf_retrieval(
                             query_embedding=query_embedding[0],
                             top_k=self.initial_docs,
                         )
+                    except EmbeddingFailedError:
+                        raise  # a degenerate query fails against every collection
                     except Exception as e:
                         logger.warning(
                             "profound_wrrf_fanout_search_failed",
@@ -1488,6 +1491,8 @@ async def _basic_vector_retrieve(
                         query_embedding=query_embedding[0],
                         top_k=self.initial_docs,
                     )
+                except EmbeddingFailedError:
+                    raise  # a degenerate query fails against every collection
                 except Exception as e:
                     logger.warning(
                         "profound_basic_fanout_search_failed",

src/perspicacite/retrieval/chroma_store.py

@@ -10,14 +10,50 @@
 except ImportError:
     IncludeEnum = None  # Will use literal values instead
 
-from perspicacite.llm.embeddings import EmbeddingProvider
+from perspicacite.llm.embeddings import EmbeddingFailedError, EmbeddingProvider, is_zero_vector
 from perspicacite.logging import get_logger
 from perspicacite.models.documents import ChunkMetadata, DocumentChunk
 from perspicacite.models.search import RetrievedChunk, SearchFilters
 
 logger = get_logger("perspicacite.retrieval.chroma")
 
 
+def _reject_degenerate_embeddings(
+    texts: list[str], embeddings: list[list[float]], collection: str
+) -> None:
+    """Refuse embeddings that would poison the collection.
+
+    Chroma accepts a zero vector and then reports cosine distance 1.0 against
+    everything, so a poisoned collection answers every query with the same
+    passages at the same score. No zero vector may be stored, whatever produced
+    it: for a text with content it means the embedder failed, and for an empty
+    text it means the chunk has nothing to retrieve.
+    """
+    if len(embeddings) != len(texts):
+        raise EmbeddingFailedError(
+            f"embedding provider returned {len(embeddings)} vector(s) for "
+            f"{len(texts)} text(s) in collection {collection!r}; refusing to "
+            "store misaligned embeddings"
+        )
+    degenerate = [
+        index
+        for index, vector in enumerate(embeddings)
+        if is_zero_vector(vector)
+    ]
+    if not degenerate:
+        return
+    first = degenerate[0]
+    reason = (
+        "the embedding provider returned a zero vector"
+        if texts[first].strip()
+        else "the chunk has no text to embed"
+    )
+    raise EmbeddingFailedError(
+        f"refusing to store {len(degenerate)} zero vector(s) in collection "
+        f"{collection!r} (first at index {first}): {reason}"
+    )
+
+
 class ChromaVectorStore:
     """
     ChromaDB-backed vector store.
@@ -177,8 +213,15 @@ def _ctx_for(i: int) -> str | None:
                 count=len(texts_to_embed),
                 collection=collection,
             )
-            embeddings = await self.embedding_provider.embed(texts_to_embed)
-            for idx, embedding in zip(indices_to_embed, embeddings):
+            try:
+                embeddings = await self.embedding_provider.embed(texts_to_embed)
+            except Exception as exc:
+                raise EmbeddingFailedError(
+                    f"embedding {len(texts_to_embed)} chunk(s) for collection "
+                    f"{collection!r} failed: {exc}"
+                ) from exc
+            _reject_degenerate_embeddings(texts_to_embed, embeddings, collection)
+            for idx, embedding in zip(indices_to_embed, embeddings, strict=True):
                 chunks[idx].embedding = embedding
 
         # Prepare data for Chroma (lists must stay aligned — never filter embeddings)
@@ -187,6 +230,20 @@ def _ctx_for(i: int) -> str | None:
         embeddings = [chunk.embedding for chunk in chunks]
         if any(e is None for e in embeddings):
             raise ValueError("All chunks must have embeddings before add_documents")
+        # Callers may arrive with chunk.embedding already set (capsule_builder,
+        # capsule_reader, local_docs embed before calling). Those vectors skip the
+        # check above, so screen everything that is about to be written.
+        poisoned = [
+            chunk.id
+            for chunk, e in zip(chunks, embeddings, strict=True)
+            if is_zero_vector(e)
+        ]
+        if poisoned:
+            raise EmbeddingFailedError(
+                f"refusing to store {len(poisoned)} zero vector(s) in collection "
+                f"{collection!r} (first chunk: {poisoned[0]!r}); a zero vector matches "
+                "every query at the same score"
+            )
         metadatas = [_chunk_to_metadata(chunk.metadata) for chunk in chunks]
 
         # Add to Chroma
@@ -229,7 +286,19 @@ async def search(
 
         Returns:
             List of retrieved chunks with scores
+
+        Raises:
+            EmbeddingFailedError: the query vector has zero norm, which Chroma
+                would answer with distance 1.0 for every document — a uniform
+                score over an arbitrary ordering rather than a ranking.
         """
+        if is_zero_vector(query_embedding):
+            raise EmbeddingFailedError(
+                f"refusing to search collection {collection!r} with a zero-norm query "
+                "embedding: every document would score identically. The embedding "
+                "provider most likely failed (check the API key and its quota)."
+            )
+
         try:
             coll = self.client.get_collection(name=collection)
         except Exception as e:

src/perspicacite/retrieval/multi_kb.py

@@ -5,6 +5,7 @@
 import asyncio
 from typing import TYPE_CHECKING, Any
 
+from perspicacite.llm.embeddings import EmbeddingFailedError
 from perspicacite.logging import get_logger
 
 if TYPE_CHECKING:
@@ -103,6 +104,10 @@ async def search(
                     query_embedding=query_embedding,
                     top_k=top_k * 2,
                 )
+            except EmbeddingFailedError:
+                # A degenerate query fails against every collection, so skipping
+                # them one by one would return an empty result reported as success.
+                raise
             except Exception as e:
                 logger.warning("multi_kb_search_collection_failed", collection=coll, error=str(e))
                 continue
@@ -252,6 +257,10 @@ async def _one(coll: str) -> list[Any]:
                 collection=coll, query_embedding=query_embedding, top_k=top_k * 2
             )
             return results
+        except EmbeddingFailedError:
+            # Degenerate query: every collection fails identically, and an empty
+            # merged result would be indistinguishable from "nothing matched".
+            raise
         except Exception as e:
             logger.warning("fanout_search_failed", collection=coll, error=str(e))
             return []

src/perspicacite/search/screening.py

@@ -8,6 +8,7 @@
 
 from rank_bm25 import BM25Plus
 
+from perspicacite.llm.embeddings import EmbeddingFailedError
 from perspicacite.logging import get_logger
 
 if TYPE_CHECKING:
@@ -487,6 +488,13 @@ async def screen_papers_embedding(
     try:
         ref_vecs = await embedding_provider.embed(list(flat))
         cand_vecs = await embedding_provider.embed(to_embed) if to_embed else []
+        # `bounds` indexes ref_vecs positionally, so a provider that drops
+        # empty texts would score candidates against the wrong paper.
+        if len(ref_vecs) != len(flat):
+            raise EmbeddingFailedError(
+                f"embedding provider returned {len(ref_vecs)} vector(s) for "
+                f"{len(flat)} reference text(s); refusing to score misaligned vectors"
+            )
     except Exception as exc:
         return [
             ScreenResult(item=c, score=0.0, kept=False, reason=f"embedding_error: {exc}")

src/perspicacite/web/app.py

@@ -30,9 +30,10 @@
 
 # Now safe to import modules that use structlog.
 from fastapi import FastAPI
-from fastapi.responses import HTMLResponse, Response
+from fastapi.responses import HTMLResponse, JSONResponse, Response
 from fastapi.staticfiles import StaticFiles
 
+from perspicacite.llm.embeddings import EmbeddingFailedError
 from perspicacite.web.routers import (
     chat as chat_router,
 )
@@ -105,6 +106,13 @@ async def _no_cache_for_assets(request, call_next):
     return response
 
 
+@app.exception_handler(EmbeddingFailedError)
+async def _embedding_failed_handler(request, exc: EmbeddingFailedError):
+    """Report a dead embedder as an upstream failure, never as a successful ingest."""
+    logger.error("Embedding failed on %s: %s", request.url.path, exc)
+    return JSONResponse(status_code=502, content={"success": False, "error": str(exc)})
+
+
 # Mount routers
 app.include_router(health_router.router)
 app.include_router(conversations_router.router)