fix: address CodeRabbit pgvector review on #979

Use can_use_pgvector_similarity for Librarian backend readiness, degrade
similarity query errors to no-match, and document deferred HNSW on Essential-1.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: northdpole

application/cmd/cre_main.py

@@ -1115,15 +1115,14 @@ def run_librarian(
 
     backend = RetrieverBackend(cfg.retriever_backend)
     if backend is RetrieverBackend.pgvector:
-        dialect = database.session.connection().dialect.name
-        if dialect != "postgresql":
+        # Readiness must include the embedding_vec column (not dialect alone):
+        # Postgres pre-migration would otherwise fail on every retrieve().
+        if not database.can_use_pgvector_similarity():
             logger.warning(
-                "CRE_LIBRARIAN_RETRIEVER_BACKEND=pgvector selected on a %r "
-                "database, but the pgvector backend needs Postgres with the "
-                "embedding_vec column (Alembic c7d8e9f0a1b2 / #977) and will "
-                "fail at retrieve() "
-                "time here. Set the backend to in_memory until then.",
-                dialect,
+                "CRE_LIBRARIAN_RETRIEVER_BACKEND=pgvector selected, but "
+                "Postgres is not ready for pgvector similarity (need Alembic "
+                "c7d8e9f0a1b2 / #977 embedding_vec). Set the backend to "
+                "in_memory until then."
             )
     # The CRE ids present in the hub are exactly the known ids the explicit
     # resolver may auto-link to (W2 seeded this from the golden set; here it is

application/database/db.py

@@ -2585,19 +2585,28 @@ def find_most_similar_embedding_id(
             "q": to_pgvector_literal(query_embedding),
             "doc_type": doc_type,
         }
-        # Prefer the Session so we stay inside the app transaction; fall back
-        # to a Connection from the Engine for SQLAlchemy 2.
-        if hasattr(self.session, "execute"):
-            row = self.session.execute(sql_text(sql), params).fetchone()
-        else:
-            with bind.connect() as conn:
-                row = conn.execute(sql_text(sql), params).fetchone()
-        if not row:
-            return None, None
-        score = float(row.score)
-        if score < similarity_threshold:
+        try:
+            # Prefer the Session so we stay inside the app transaction; fall
+            # back to a Connection from the Engine for SQLAlchemy 2.
+            if hasattr(self.session, "execute"):
+                row = self.session.execute(sql_text(sql), params).fetchone()
+            else:
+                with bind.connect() as conn:
+                    row = conn.execute(sql_text(sql), params).fetchone()
+            if not row:
+                return None, None
+            score = float(row.score)
+            if score < similarity_threshold:
+                return None, None
+            return str(row.object_id), score
+        except Exception as exc:
+            # Match prior sklearn-path resilience: chat/import should degrade
+            # to "no match" rather than 500 on a transient driver/query error.
+            logger.warning(
+                "pgvector similarity query failed (%s); returning no match",
+                exc,
+            )
             return None, None
-        return str(row.object_id), score
 
     def assert_embedding_contract(
         self,

application/database/pgvector_utils.py

@@ -5,6 +5,9 @@
 only where there is enough RAM (local or a high-resource worker), then
 sync/backfill vectors. Alembic on prod only: enable extension, add column,
 copy CSV→vector for matching dims.
+
+HNSW/IVFFlat indexes are deferred (Essential-1 capacity); see migration
+``c7d8e9f0a1b2`` docstring for the follow-up plan.
 """
 
 from __future__ import annotations

application/tests/prompt_client_pgvector_similarity_test.py

@@ -1,7 +1,7 @@
 """Chat/import similarity prefers pgvector when the DB reports it ready."""
 
 import unittest
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
 from application.prompt_client import prompt_client as prompt_client_mod
 
@@ -44,5 +44,26 @@ def test_cre_paginated_uses_pgvector_when_available(self) -> None:
         self.assertEqual(kwargs["id_column"], "cre_id")
 
 
+class FindMostSimilarEmbeddingIdResilienceTest(unittest.TestCase):
+    def test_query_error_returns_no_match(self) -> None:
+        from application.database.db import Node_collection
+
+        database = Node_collection.__new__(Node_collection)
+        session = MagicMock()
+        session.execute.side_effect = RuntimeError("driver boom")
+        session.get_bind.return_value = MagicMock()
+        database.session = session
+
+        with patch("application.database.db.logger"):
+            match_id, score = database.find_most_similar_embedding_id(
+                [0.1, 0.2, 0.3],
+                doc_type="CRE",
+                id_column="cre_id",
+                similarity_threshold=0.7,
+            )
+        self.assertIsNone(match_id)
+        self.assertIsNone(score)
+
+
 if __name__ == "__main__":
     unittest.main()

migrations/versions/c7d8e9f0a1b2_add_embedding_vec_pgvector.py

@@ -9,6 +9,11 @@
 OOM-killed. This migration only: CREATE EXTENSION vector, ADD COLUMN
 embedding_vec, and copy matching-dim CSV → vector. Fix missing / wrong-dim
 rows offline on a high-RAM machine, then re-run / backfill.
+
+ANN index (HNSW/IVFFlat) is intentionally deferred: opencreorg runs on
+Heroku Postgres Essential-1; add a follow-up migration with
+``CREATE INDEX ... USING hnsw (embedding_vec vector_cosine_ops)`` once
+dims/backfill are settled and plan capacity allows.
 """
 
 from alembic import op