Merge pull request #1970 from Open-Source-Legal/fix/fts-natural-language-recall

Fix FTS arm recall for natural-language queries (OR lexemes, not AND)

Author: JSv4

changelog.d/fts-natural-language-recall.fixed.md

@@ -0,0 +1 @@
+- **Full-text search recall for natural-language queries** (`opencontractserver/llms/vector_stores/core_vector_stores.py`, `_run_fts_query`). The FTS arm built its query with `plainto_tsquery` (via `SearchQuery`'s default), which ANDs every lexeme — so a multi-concept question like *"What are the main risks or obligations across these documents?"* matched **no** annotation (no single row contains all terms) and the arm returned zero rows. When a corpus has no embeddings, hybrid search degraded to that empty FTS arm and the agent answered "I found nothing." `_run_fts_query` now OR-combines per-token `plainto_tsquery` sub-queries, so any term match qualifies while `SearchRank` still ranks annotations matching more terms higher. Per-token `plainto` safely elides stopwords without raising a tsquery syntax error (no manual stopword list or raw-tsquery escaping). This is the single FTS entry point for the hybrid, fts-only, and `global_search` paths, so the recall fix applies across all of them. Measured on a 53-document SEC-filing corpus: the example question went from 0 → 8 full-text matches. Regression test: `test_hybrid_search.py::TestHybridSearch::test_fts_matches_natural_language_query_spanning_annotations`.

opencontractserver/llms/vector_stores/core_vector_stores.py

@@ -2,7 +2,10 @@
 
 import asyncio
 import logging
+import operator
+import re
 from dataclasses import dataclass, field
+from functools import reduce
 from typing import Any, Literal, Optional, Union
 
 from asgiref.sync import async_to_sync, sync_to_async
@@ -1096,7 +1099,24 @@ def _run_fts_query(queryset, query_text: str, oversample_k: int) -> list:
         """
         from django.contrib.postgres.search import SearchQuery, SearchRank
 
-        search_query = SearchQuery(query_text, config=FTS_CONFIG)
+        # plainto_tsquery (SearchQuery's default) ANDs every lexeme, so a
+        # multi-concept natural-language question ("risks AND obligations AND
+        # documents ...") matches no single annotation and the FTS arm returns
+        # nothing — leaving a corpus without embeddings at zero recall. OR-
+        # combine per-token plainto queries instead: any term match qualifies,
+        # while SearchRank still ranks annotations matching more terms higher.
+        # Per-token plainto safely elides stopwords without raising a tsquery
+        # syntax error, so no manual stopword list (or raw-tsquery escaping) is
+        # needed. This is the single entry point for every FTS arm (hybrid,
+        # fts-only, and global_search), so the recall fix applies everywhere.
+        tokens = re.findall(r"\w+", query_text)
+        if tokens:
+            search_query = reduce(
+                operator.or_,
+                (SearchQuery(token, config=FTS_CONFIG) for token in tokens),
+            )
+        else:
+            search_query = SearchQuery(query_text, config=FTS_CONFIG)
         text_qs = (
             queryset.filter(search_vector=search_query)
             .annotate(text_rank=SearchRank("search_vector", search_query))

opencontractserver/tests/test_hybrid_search.py

@@ -184,6 +184,41 @@ def test_hybrid_search_text_only_fallback(self, mock_embed):
         # Text-only arm should still find annotations via search_vector
         self.assertIsInstance(results, list)
 
+    def test_fts_matches_natural_language_query_spanning_annotations(self):
+        """FTS arm must match a natural-language query whose content terms are
+        spread across several annotations.
+
+        The three fixture annotations each carry a *different* concept
+        ("indemnification", "termination", "force majeure ... obligations").
+        A real user question references more than one of them at once. With
+        ``plainto_tsquery`` (AND of every lexeme) no single annotation
+        contains them all, so the FTS arm returns nothing and a corpus with
+        no embeddings answers "I found nothing". OR-ing the content lexemes
+        keeps recall: each annotation that matches *any* term is returned,
+        and ``SearchRank`` still favours annotations matching more terms.
+
+        Isolated to ``mode="fts"`` so this asserts the full-text arm alone,
+        independent of the vector arm / embeddings.
+        """
+        store = self._make_store()
+        query = VectorSearchQuery(
+            query_text=(
+                "What indemnification, termination, and force majeure "
+                "obligations apply?"
+            ),
+            similarity_top_k=10,
+            mode="fts",
+        )
+        results = store.search(query)
+        matched = {r.annotation.id for r in results}
+        self.assertEqual(
+            matched,
+            {self.anno_alpha.id, self.anno_beta.id, self.anno_gamma.id},
+            "FTS should return every annotation whose terms collectively "
+            "satisfy the natural-language query (OR semantics), not zero "
+            "(AND semantics from plainto_tsquery).",
+        )
+
     @patch(
         "opencontractserver.llms.vector_stores.base_vector_store"
         ".agenerate_embeddings_from_text"