diff --git a/.env.example b/.env.example index 6a981d63ec..54246577ca 100644 --- a/.env.example +++ b/.env.example @@ -115,6 +115,11 @@ HINDSIGHT_API_LOG_LEVEL=info # chinese_lindera/lindera(chinese), japanese_lindera/lindera(japanese), # korean_lindera/lindera(korean), ngram(min,max), edge_ngram(min,max) # HINDSIGHT_API_TEXT_SEARCH_EXTENSION_PG_SEARCH_TOKENIZER= +# Optional cap on the number of terms in the native PostgreSQL BM25 tsquery. +# Long queries OR-join every normalized token, which can match too much of a +# large bank. 0 (default) keeps the historical uncapped behavior; a positive +# value bounds only the native backend (other BM25 backends get the raw query). +# HINDSIGHT_API_BM25_MAX_QUERY_TERMS=0 # File Parser (Optional - uses markitdown by default) # HINDSIGHT_API_FILE_PARSER=markitdown diff --git a/hindsight-api-slim/hindsight_api/config.py b/hindsight-api-slim/hindsight_api/config.py index 44d2e9e81e..1d88d43376 100644 --- a/hindsight-api-slim/hindsight_api/config.py +++ b/hindsight-api-slim/hindsight_api/config.py @@ -638,6 +638,7 @@ def _resolve_operation_temperature(operation_env: str, default: float) -> float # Recall candidate gating (per-source cap + BM25 score floor) ENV_BM25_MIN_SCORE = "HINDSIGHT_API_BM25_MIN_SCORE" +ENV_BM25_MAX_QUERY_TERMS = "HINDSIGHT_API_BM25_MAX_QUERY_TERMS" ENV_RECALL_MAX_CANDIDATES_PER_SOURCE = "HINDSIGHT_API_RECALL_MAX_CANDIDATES_PER_SOURCE" # Per-strategy recall boost. Prioritises specific retrieval arms (semantic, # bm25, graph, temporal) on recall via a human priority level — e.g. @@ -789,6 +790,9 @@ def _resolve_operation_temperature(operation_env: str, default: float) -> float # zero-score (non-matching) rows on backends — notably VectorChord — whose # operator ranks every document rather than pre-filtering to term matches. DEFAULT_BM25_MIN_SCORE = 0.0 +# Native tsvector BM25 can optionally cap the OR tsquery built from normalized +# query tokens. 0 preserves the historical uncapped behavior. +DEFAULT_BM25_MAX_QUERY_TERMS = 0 # Per-source candidate cap applied to each retrieval arm (semantic, BM25, graph, # temporal) before RRF, so a single over-expanding backend cannot fill the # reranker's global candidate budget on its own. 0 disables the cap. @@ -1209,6 +1213,19 @@ def _parse_positive_int(name: str, raw: str | None, default: int) -> int: return parsed +def _parse_non_negative_int(name: str, raw: str | None, default: int) -> int: + """Parse an env var that must be an integer >= 0.""" + if raw is None or raw == "": + return default + try: + parsed = int(raw) + except ValueError as e: + raise ValueError(f"{name} must be an integer, got {raw!r}") from e + if parsed < 0: + raise ValueError(f"{name} must be >= 0, got {parsed}") + return parsed + + def _parse_optional_positive_int(name: str, raw: str | None) -> int | None: """Parse an optional env var that must be a positive integer when set.""" if raw is None or raw == "": @@ -1979,6 +1996,7 @@ class HindsightConfig: reflect_llm_strategy: LLMStrategyConfig | None = None consolidation_llm_members: list[LLMMemberConfig] = field(default_factory=list) consolidation_llm_strategy: LLMStrategyConfig | None = None + bm25_max_query_terms: int = DEFAULT_BM25_MAX_QUERY_TERMS # Class-level sets for configuration categorization @@ -2179,6 +2197,9 @@ def validate(self) -> None: f"Invalid semantic_min_similarity: {self.semantic_min_similarity}. Must be between 0.0 and 1.0" ) + if self.bm25_max_query_terms < 0: + raise ValueError(f"Invalid bm25_max_query_terms: {self.bm25_max_query_terms}. Must be >= 0") + # Validate bedrock_service_tier valid_bedrock_tiers = (None, "flex", "priority", "reserved") if self.llm_bedrock_service_tier not in valid_bedrock_tiers: @@ -2608,6 +2629,11 @@ def from_env(cls) -> "HindsightConfig": reranker_max_candidates=int(os.getenv(ENV_RERANKER_MAX_CANDIDATES, str(DEFAULT_RERANKER_MAX_CANDIDATES))), semantic_min_similarity=float(os.getenv(ENV_SEMANTIC_MIN_SIMILARITY, str(DEFAULT_SEMANTIC_MIN_SIMILARITY))), bm25_min_score=float(os.getenv(ENV_BM25_MIN_SCORE, str(DEFAULT_BM25_MIN_SCORE))), + bm25_max_query_terms=_parse_non_negative_int( + ENV_BM25_MAX_QUERY_TERMS, + os.getenv(ENV_BM25_MAX_QUERY_TERMS), + DEFAULT_BM25_MAX_QUERY_TERMS, + ), recall_max_candidates_per_source=int( os.getenv(ENV_RECALL_MAX_CANDIDATES_PER_SOURCE, str(DEFAULT_RECALL_MAX_CANDIDATES_PER_SOURCE)) ), diff --git a/hindsight-api-slim/hindsight_api/engine/search/retrieval.py b/hindsight-api-slim/hindsight_api/engine/search/retrieval.py index 56363a04e7..37e0cfabaf 100644 --- a/hindsight-api-slim/hindsight_api/engine/search/retrieval.py +++ b/hindsight-api-slim/hindsight_api/engine/search/retrieval.py @@ -15,7 +15,7 @@ from datetime import UTC, datetime from typing import TYPE_CHECKING, Any, Optional -from ...config import get_config +from ...config import DEFAULT_BM25_MAX_QUERY_TERMS, get_config from ..db_utils import acquire_with_retry from ..memory_engine import fq_table from ..sql import create_sql_dialect @@ -222,7 +222,12 @@ async def retrieve_semantic_bm25_combined( # --- BM25 UNION ALL arms (one per fact_type, only when tokens present) --- if _include_bm25: text_ext = config.text_search_extension - bm25_text_param: str = dialect.prepare_bm25_text(tokens, query_text, text_search_extension=text_ext) + bm25_text_param: str = dialect.prepare_bm25_text( + tokens, + query_text, + text_search_extension=text_ext, + max_query_terms=getattr(config, "bm25_max_query_terms", DEFAULT_BM25_MAX_QUERY_TERMS), + ) for i, ft in enumerate(fact_types): arms.append( dialect.build_bm25_arm( diff --git a/hindsight-api-slim/hindsight_api/engine/sql/base.py b/hindsight-api-slim/hindsight_api/engine/sql/base.py index 92b1ea9430..95ea991a2e 100644 --- a/hindsight-api-slim/hindsight_api/engine/sql/base.py +++ b/hindsight-api-slim/hindsight_api/engine/sql/base.py @@ -449,6 +449,7 @@ def prepare_bm25_text( query_text: str, *, text_search_extension: str = "native", + max_query_terms: int | None = None, ) -> str: """Prepare the text parameter value for BM25 search. @@ -459,6 +460,8 @@ def prepare_bm25_text( tokens: Tokenized query words. query_text: Original query text. text_search_extension: Full-text search backend variant. + max_query_terms: Optional backend-specific token cap. 0 or None + leaves query terms uncapped. Returns: Prepared text string to bind as the BM25 text parameter. diff --git a/hindsight-api-slim/hindsight_api/engine/sql/oracle.py b/hindsight-api-slim/hindsight_api/engine/sql/oracle.py index 3f8610b072..e5fdcb73bf 100644 --- a/hindsight-api-slim/hindsight_api/engine/sql/oracle.py +++ b/hindsight-api-slim/hindsight_api/engine/sql/oracle.py @@ -303,6 +303,7 @@ def prepare_bm25_text( query_text: str, *, text_search_extension: str = "native", + max_query_terms: int | None = None, ) -> str: # Oracle Text: filter tokens with special chars, escape reserved words # with curly braces (e.g. "about" → "{about}"), and join with OR. diff --git a/hindsight-api-slim/hindsight_api/engine/sql/postgresql.py b/hindsight-api-slim/hindsight_api/engine/sql/postgresql.py index 42ac27cf37..d0a7b0bf05 100644 --- a/hindsight-api-slim/hindsight_api/engine/sql/postgresql.py +++ b/hindsight-api-slim/hindsight_api/engine/sql/postgresql.py @@ -254,8 +254,11 @@ def prepare_bm25_text( query_text: str, *, text_search_extension: str = "native", + max_query_terms: int | None = None, ) -> str: if text_search_extension in ("vchord", "pg_textsearch", "pgroonga", "pg_search"): return query_text + if max_query_terms is not None and max_query_terms > 0: + tokens = tokens[:max_query_terms] # native tsvector: join tokens with OR operator return " | ".join(tokens) diff --git a/hindsight-api-slim/tests/test_multilingual_bm25.py b/hindsight-api-slim/tests/test_multilingual_bm25.py index 8a63cb292a..23db7b5dba 100644 --- a/hindsight-api-slim/tests/test_multilingual_bm25.py +++ b/hindsight-api-slim/tests/test_multilingual_bm25.py @@ -10,12 +10,18 @@ from __future__ import annotations from pathlib import Path +from types import SimpleNamespace from unittest.mock import MagicMock +import pytest + from hindsight_api.engine.consolidation.prompts import build_batch_consolidation_prompt from hindsight_api.engine.prompt_utils import output_language_directive from hindsight_api.engine.reflect.prompts import build_final_system_prompt from hindsight_api.engine.retain.fact_extraction import _build_extraction_prompt_and_schema +from hindsight_api.engine.search import retrieval as retrieval_mod +from hindsight_api.engine.search.retrieval import tokenize_query +from hindsight_api.engine.sql.postgresql import PostgreSQLDialect def _baseline_config() -> MagicMock: @@ -165,3 +171,75 @@ def test_configurable_bm25_language_migration_chains_off_head(): src = target.read_text() assert 'revision: str = "p4q5r6s7t8u9"' in src assert 'down_revision: str | Sequence[str] | None = "86f7a033d372"' in src + + +# --------------------------------------------------------------------------- +# BM25 query term cap +# --------------------------------------------------------------------------- + + +def test_postgresql_native_bm25_caps_raw_terms_preserving_order(): + query = "Alpha beta alpha, gamma delta beta epsilon" + tokens = tokenize_query(query) + + assert PostgreSQLDialect().prepare_bm25_text(tokens, query, max_query_terms=3) == "alpha | beta | alpha" + + +def test_postgresql_native_bm25_zero_cap_keeps_existing_unlimited_behavior(): + query = "Alpha beta alpha" + tokens = tokenize_query(query) + + assert PostgreSQLDialect().prepare_bm25_text(tokens, query, max_query_terms=0) == "alpha | beta | alpha" + + +def test_postgresql_extension_bm25_keeps_raw_query_text(): + query = "Alpha beta alpha, gamma delta beta epsilon" + tokens = tokenize_query(query) + + assert ( + PostgreSQLDialect().prepare_bm25_text(tokens, query, text_search_extension="vchord", max_query_terms=3) == query + ) + + +@pytest.mark.asyncio +async def test_combined_retrieval_uses_default_bm25_cap_for_legacy_config(monkeypatch): + class FakeDialect: + max_query_terms: int | None = None + + def build_semantic_arm(self, **kwargs): + return "SELECT 'semantic' AS source" + + def build_bm25_arm(self, **kwargs): + return "SELECT 'bm25' AS source" + + def prepare_bm25_text(self, tokens, query_text, *, text_search_extension="native", max_query_terms=None): + self.max_query_terms = max_query_terms + return " | ".join(tokens) + + class FakeConn: + backend_type = "postgresql" + + async def fetch(self, query, *params): + return [] + + fake_dialect = FakeDialect() + legacy_config = SimpleNamespace( + semantic_min_similarity=0.0, + bm25_min_score=0.0, + text_search_extension="native", + text_search_extension_native_language="english", + ) + monkeypatch.setattr(retrieval_mod, "get_config", lambda: legacy_config) + monkeypatch.setattr(retrieval_mod, "create_sql_dialect", lambda backend: fake_dialect) + + result = await retrieval_mod.retrieve_semantic_bm25_combined( + FakeConn(), + "[0.0]", + "alpha beta", + "bank-1", + ["observation"], + 5, + ) + + assert result == {"observation": ([], [])} + assert fake_dialect.max_query_terms == 0 diff --git a/hindsight-api-slim/tests/test_recall_config.py b/hindsight-api-slim/tests/test_recall_config.py index f47f6a1726..3d5ba2b6e3 100644 --- a/hindsight-api-slim/tests/test_recall_config.py +++ b/hindsight-api-slim/tests/test_recall_config.py @@ -65,12 +65,14 @@ class TestRecallConfigFields: """Hierarchical config fields for internal recall.""" def test_fields_exist_on_dataclass(self): - from hindsight_api.config import HindsightConfig + from hindsight_api.config import DEFAULT_BM25_MAX_QUERY_TERMS, HindsightConfig names = {f.name for f in dataclasses.fields(HindsightConfig)} assert "recall_include_chunks" in names assert "recall_max_tokens" in names assert "recall_chunks_max_tokens" in names + assert "bm25_max_query_terms" in names + assert HindsightConfig.__dataclass_fields__["bm25_max_query_terms"].default == DEFAULT_BM25_MAX_QUERY_TERMS def test_fields_are_configurable(self): from hindsight_api.config import HindsightConfig @@ -82,6 +84,7 @@ def test_fields_are_configurable(self): def test_default_values(self): from hindsight_api.config import ( + DEFAULT_BM25_MAX_QUERY_TERMS, DEFAULT_RECALL_CHUNKS_MAX_TOKENS, DEFAULT_RECALL_INCLUDE_CHUNKS, DEFAULT_RECALL_MAX_TOKENS, @@ -90,9 +93,11 @@ def test_default_values(self): assert DEFAULT_RECALL_INCLUDE_CHUNKS is True assert DEFAULT_RECALL_MAX_TOKENS == 2048 assert DEFAULT_RECALL_CHUNKS_MAX_TOKENS == 1000 + assert DEFAULT_BM25_MAX_QUERY_TERMS == 0 def test_env_var_constants(self): from hindsight_api.config import ( + ENV_BM25_MAX_QUERY_TERMS, ENV_RECALL_CHUNKS_MAX_TOKENS, ENV_RECALL_INCLUDE_CHUNKS, ENV_RECALL_MAX_TOKENS, @@ -101,6 +106,7 @@ def test_env_var_constants(self): assert ENV_RECALL_INCLUDE_CHUNKS == "HINDSIGHT_API_RECALL_INCLUDE_CHUNKS" assert ENV_RECALL_MAX_TOKENS == "HINDSIGHT_API_RECALL_MAX_TOKENS" assert ENV_RECALL_CHUNKS_MAX_TOKENS == "HINDSIGHT_API_RECALL_CHUNKS_MAX_TOKENS" + assert ENV_BM25_MAX_QUERY_TERMS == "HINDSIGHT_API_BM25_MAX_QUERY_TERMS" @patch.dict( "os.environ", @@ -108,6 +114,7 @@ def test_env_var_constants(self): "HINDSIGHT_API_RECALL_INCLUDE_CHUNKS": "false", "HINDSIGHT_API_RECALL_MAX_TOKENS": "777", "HINDSIGHT_API_RECALL_CHUNKS_MAX_TOKENS": "333", + "HINDSIGHT_API_BM25_MAX_QUERY_TERMS": "24", }, ) def test_from_env_reads_overrides(self): @@ -117,6 +124,14 @@ def test_from_env_reads_overrides(self): assert config.recall_include_chunks is False assert config.recall_max_tokens == 777 assert config.recall_chunks_max_tokens == 333 + assert config.bm25_max_query_terms == 24 + + @patch.dict("os.environ", {"HINDSIGHT_API_BM25_MAX_QUERY_TERMS": "-1"}) + def test_from_env_rejects_negative_bm25_max_query_terms(self): + from hindsight_api.config import HindsightConfig + + with pytest.raises(ValueError, match="HINDSIGHT_API_BM25_MAX_QUERY_TERMS must be >= 0"): + HindsightConfig.from_env() class TestMentalModelTriggerRecallFields: diff --git a/hindsight-docs/docs/developer/configuration.md b/hindsight-docs/docs/developer/configuration.md index 52cb8e4bbc..02e35f24e1 100644 --- a/hindsight-docs/docs/developer/configuration.md +++ b/hindsight-docs/docs/developer/configuration.md @@ -145,6 +145,7 @@ If you need to switch from one extension to another: | `HINDSIGHT_API_TEXT_SEARCH_EXTENSION` | Text search backend: `native`, `vchord`, `pg_textsearch`, `pgroonga`, or `pg_search` | `native` | | `HINDSIGHT_API_TEXT_SEARCH_EXTENSION_NATIVE_LANGUAGE` | PostgreSQL text search dictionary used by the `native` backend (e.g. `english`, `french`, `simple`, `zhparser`) | `english` | | `HINDSIGHT_API_TEXT_SEARCH_EXTENSION_PG_SEARCH_TOKENIZER` | ParadeDB `pg_search` tokenizer used when creating BM25 indexes. Empty uses ParadeDB's default tokenizer (`unicode_words`). | unset | +| `HINDSIGHT_API_BM25_MAX_QUERY_TERMS` | Optional cap on the number of terms in the native PostgreSQL BM25 `tsquery`. Long queries OR-join every normalized token, which can match too much of a large bank. `0` keeps the historical uncapped behavior; a positive value bounds only the `native` backend (other BM25 backends receive the raw query). | `0` | | `HINDSIGHT_API_LLM_OUTPUT_LANGUAGE` | When set, forces every LLM-generated artifact (retain facts, consolidation observations, reflect responses) into this language. Free-form (e.g. `Spanish`, `Japanese`). | unset | Hindsight supports five backends for BM25 keyword retrieval: diff --git a/hindsight-embed/hindsight_embed/env.example b/hindsight-embed/hindsight_embed/env.example index 6a981d63ec..54246577ca 100644 --- a/hindsight-embed/hindsight_embed/env.example +++ b/hindsight-embed/hindsight_embed/env.example @@ -115,6 +115,11 @@ HINDSIGHT_API_LOG_LEVEL=info # chinese_lindera/lindera(chinese), japanese_lindera/lindera(japanese), # korean_lindera/lindera(korean), ngram(min,max), edge_ngram(min,max) # HINDSIGHT_API_TEXT_SEARCH_EXTENSION_PG_SEARCH_TOKENIZER= +# Optional cap on the number of terms in the native PostgreSQL BM25 tsquery. +# Long queries OR-join every normalized token, which can match too much of a +# large bank. 0 (default) keeps the historical uncapped behavior; a positive +# value bounds only the native backend (other BM25 backends get the raw query). +# HINDSIGHT_API_BM25_MAX_QUERY_TERMS=0 # File Parser (Optional - uses markitdown by default) # HINDSIGHT_API_FILE_PARSER=markitdown