From 7c49ff3a1ad6d773b977fa5f121de3ec9fcda7c1 Mon Sep 17 00:00:00 2001 From: JSv4 Date: Tue, 9 Jun 2026 07:21:55 -0500 Subject: [PATCH] Fix FTS arm recall for natural-language queries (OR lexemes) The full-text arm of CoreAnnotationVectorStore built its query with plainto_tsquery (SearchQuery's default), which ANDs every lexeme. A multi-concept natural-language question therefore matched no single annotation and the FTS arm returned zero rows -- so a corpus with no embeddings (hybrid degrading to FTS-only) answered 'I found nothing'. _run_fts_query now OR-combines per-token plainto_tsquery sub-queries: any term match qualifies, while SearchRank still ranks annotations matching more terms higher. Per-token plainto safely elides stopwords without raising a tsquery syntax error, so no manual stopword list or raw-tsquery escaping is needed. This is the single FTS entry point for the hybrid, fts-only and global_search paths, so the fix applies to all. Added test_fts_matches_natural_language_query_spanning_annotations (RED before the change: 0 matches; GREEN after: all three annotations whose terms collectively satisfy the query). --- .../fts-natural-language-recall.fixed.md | 1 + .../llms/vector_stores/core_vector_stores.py | 22 +++++++++++- .../tests/test_hybrid_search.py | 35 +++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fts-natural-language-recall.fixed.md diff --git a/changelog.d/fts-natural-language-recall.fixed.md b/changelog.d/fts-natural-language-recall.fixed.md new file mode 100644 index 000000000..5ec59a1d3 --- /dev/null +++ b/changelog.d/fts-natural-language-recall.fixed.md @@ -0,0 +1 @@ +- **Full-text search recall for natural-language queries** (`opencontractserver/llms/vector_stores/core_vector_stores.py`, `_run_fts_query`). The FTS arm built its query with `plainto_tsquery` (via `SearchQuery`'s default), which ANDs every lexeme — so a multi-concept question like *"What are the main risks or obligations across these documents?"* matched **no** annotation (no single row contains all terms) and the arm returned zero rows. When a corpus has no embeddings, hybrid search degraded to that empty FTS arm and the agent answered "I found nothing." `_run_fts_query` now OR-combines per-token `plainto_tsquery` sub-queries, so any term match qualifies while `SearchRank` still ranks annotations matching more terms higher. Per-token `plainto` safely elides stopwords without raising a tsquery syntax error (no manual stopword list or raw-tsquery escaping). This is the single FTS entry point for the hybrid, fts-only, and `global_search` paths, so the recall fix applies across all of them. Measured on a 53-document SEC-filing corpus: the example question went from 0 → 8 full-text matches. Regression test: `test_hybrid_search.py::TestHybridSearch::test_fts_matches_natural_language_query_spanning_annotations`. diff --git a/opencontractserver/llms/vector_stores/core_vector_stores.py b/opencontractserver/llms/vector_stores/core_vector_stores.py index 988971de5..39dc89672 100644 --- a/opencontractserver/llms/vector_stores/core_vector_stores.py +++ b/opencontractserver/llms/vector_stores/core_vector_stores.py @@ -2,7 +2,10 @@ import asyncio import logging +import operator +import re from dataclasses import dataclass, field +from functools import reduce from typing import Any, Literal, Optional, Union from asgiref.sync import async_to_sync, sync_to_async @@ -1096,7 +1099,24 @@ def _run_fts_query(queryset, query_text: str, oversample_k: int) -> list: """ from django.contrib.postgres.search import SearchQuery, SearchRank - search_query = SearchQuery(query_text, config=FTS_CONFIG) + # plainto_tsquery (SearchQuery's default) ANDs every lexeme, so a + # multi-concept natural-language question ("risks AND obligations AND + # documents ...") matches no single annotation and the FTS arm returns + # nothing — leaving a corpus without embeddings at zero recall. OR- + # combine per-token plainto queries instead: any term match qualifies, + # while SearchRank still ranks annotations matching more terms higher. + # Per-token plainto safely elides stopwords without raising a tsquery + # syntax error, so no manual stopword list (or raw-tsquery escaping) is + # needed. This is the single entry point for every FTS arm (hybrid, + # fts-only, and global_search), so the recall fix applies everywhere. + tokens = re.findall(r"\w+", query_text) + if tokens: + search_query = reduce( + operator.or_, + (SearchQuery(token, config=FTS_CONFIG) for token in tokens), + ) + else: + search_query = SearchQuery(query_text, config=FTS_CONFIG) text_qs = ( queryset.filter(search_vector=search_query) .annotate(text_rank=SearchRank("search_vector", search_query)) diff --git a/opencontractserver/tests/test_hybrid_search.py b/opencontractserver/tests/test_hybrid_search.py index 6f5eab677..73fb06d08 100644 --- a/opencontractserver/tests/test_hybrid_search.py +++ b/opencontractserver/tests/test_hybrid_search.py @@ -184,6 +184,41 @@ def test_hybrid_search_text_only_fallback(self, mock_embed): # Text-only arm should still find annotations via search_vector self.assertIsInstance(results, list) + def test_fts_matches_natural_language_query_spanning_annotations(self): + """FTS arm must match a natural-language query whose content terms are + spread across several annotations. + + The three fixture annotations each carry a *different* concept + ("indemnification", "termination", "force majeure ... obligations"). + A real user question references more than one of them at once. With + ``plainto_tsquery`` (AND of every lexeme) no single annotation + contains them all, so the FTS arm returns nothing and a corpus with + no embeddings answers "I found nothing". OR-ing the content lexemes + keeps recall: each annotation that matches *any* term is returned, + and ``SearchRank`` still favours annotations matching more terms. + + Isolated to ``mode="fts"`` so this asserts the full-text arm alone, + independent of the vector arm / embeddings. + """ + store = self._make_store() + query = VectorSearchQuery( + query_text=( + "What indemnification, termination, and force majeure " + "obligations apply?" + ), + similarity_top_k=10, + mode="fts", + ) + results = store.search(query) + matched = {r.annotation.id for r in results} + self.assertEqual( + matched, + {self.anno_alpha.id, self.anno_beta.id, self.anno_gamma.id}, + "FTS should return every annotation whose terms collectively " + "satisfy the natural-language query (OR semantics), not zero " + "(AND semantics from plainto_tsquery).", + ) + @patch( "opencontractserver.llms.vector_stores.base_vector_store" ".agenerate_embeddings_from_text"