LCORE-1853: Add relevance cutoff score to inline BYOK RAG

Author: syedriko

docs/byok_guide.md

@@ -79,6 +79,7 @@ Both modes rely on:
 
 Inline RAG additionally supports:
 - **Score Multiplier**: Optional weight applied per BYOK vector store when mixing multiple sources. Allows custom prioritization of content.
+- **Relevance cutoff score** (`relevance_cutoff_score` in `byok_rag`): Minimum raw similarity score for a chunk to be returned from that BYOK vector store. Chunks below the threshold are dropped before results are merged and ranked with other sources. Configure per knowledge source (each `byok_rag` entry has its own value). The default when omitted is `0.3` (see `DEFAULT_BYOK_RAG_RELEVANCE_CUTOFF_SCORE` in `src/constants.py`). This value is passed to Llama Stack as the vector search `score_threshold` for that store.
 
 > [!NOTE]
 > OKP and BYOK scores are not directly comparable (different scoring systems), so
@@ -89,6 +90,13 @@ Inline RAG additionally supports:
 > chunks (BYOK + OKP) delivered to the LLM. Tool RAG is controlled independently
 > by `TOOL_RAG_MAX_CHUNKS`.
 
+
+> [!NOTE]
+> `relevance_cutoff_score` applies to Inline RAG only. When the model uses Tool RAG (`file_search`),
+> Lightspeed Stack does not send this setting; retrieval uses Llama Stack’s default ranking for that path.
+> Use Inline RAG if you need per-store cutoff behavior from configuration.
+
+
 ---
 
 ## Prerequisites
@@ -236,8 +244,13 @@ byok_rag:
     vector_db_id: vs_d4c8e1f0-92ab-4d3c-a5e7-6b8f0c2d1e3a
     db_path: /data/vector_dbs/internal_kb/faiss_store.db
     score_multiplier: 1.2       # Boost results from this store
+    relevance_cutoff_score: 0.3  # Optional: min raw similarity per chunk for this store (Inline RAG only; default 0.3)    
 ```
 
+`relevance_cutoff_score` is interpreted in the same score space as the vector backend for that
+store. It is not comparable across different vector stores or OKP; tune each `byok_rag` entry
+using retrieval quality on that corpus.
+
 **⚠️ Important**: The `vector_db_id` value must exactly match the ID generated by the rag-content tool during index creation (e.g. `vs_8c94967b-81cc-4028-a294-9cfac6fd9ae2`). This identifier links your configuration to the specific vector database index.
 
 ### Step 5: Configure RAG Strategy
@@ -265,10 +278,10 @@ okp:
 
 Both modes can be enabled simultaneously. Choose based on your latency and control preferences:
 
-| Mode       | When context is fetched | Tool call needed | score_multiplier |
-|------------|-------------------------|------------------|------------------|
-| Inline RAG | With every query        | No               | Yes (BYOK only)  |
-| Tool RAG   | On LLM demand           | Yes              | No               |
+| Mode | When context is fetched | Tool call needed | score_multiplier | relevance_cutoff_score |
+|------|------------------------|------------------|----------------|------------------------|
+| Inline RAG | With every query | No | Yes (BYOK only) | Yes (BYOK only) |
+| Tool RAG | On LLM demand | Yes | No | No  |
 
 > [!TIP]
 > A ready-to-use example combining BYOK and OKP is available at

docs/openapi.json

@@ -11870,6 +11870,13 @@
                         "description": "Multiplier applied to relevance scores from this vector store. Used to weight results when querying multiple knowledge sources. Values > 1 boost this store's results; values < 1 reduce them.",
                         "default": 1.0
                     },
+                    "relevance_cutoff_score": {
+                        "type": "number",
+                        "exclusiveMinimum": 0.0,
+                        "title": "Relevance cutoff score",
+                        "description": "Minimum raw similarity score to consider a result relevant. Results with a similarity score below this threshold are not returned.",
+                        "default": 0.3
+                    },
                     "host": {
                         "anyOf": [
                             {

pyproject.toml

@@ -229,6 +229,10 @@ ignore = ["query.py"]
 requires = ["pdm-backend"]
 build-backend = "pdm.backend"
 
+[tool.pylint.design]
+max-args = 8
+max-positional-arguments = 8
+
 [tool.pylint."MESSAGES CONTROL"]
 disable = ["R0801"]
 

src/configuration.py

@@ -547,6 +547,24 @@ def score_multiplier_mapping(self) -> dict[str, float]:
             for brag in self._configuration.byok_rag
         }
 
+    @property
+    def relevance_cutoff_mapping(self) -> dict[str, float]:
+        """Return mapping from vector_db_id to relevance_cutoff_score from BYOK RAG config.
+
+        Returns:
+            dict[str, float]: Mapping where keys are llama-stack vector_db_ids
+            and values are relevance cutoff scores from configuration.
+
+        Raises:
+            LogicError: If the configuration has not been loaded.
+        """
+        if self._configuration is None:
+            raise LogicError("logic error: configuration is not loaded")
+        return {
+            brag.vector_db_id: brag.relevance_cutoff_score
+            for brag in self._configuration.byok_rag
+        }
+
     @property
     def inline_solr_enabled(self) -> bool:
         """Return whether OKP is included in the inline RAG list.

src/constants.py

@@ -203,6 +203,9 @@
 # Score multiplier applied to BYOK chunks after cross-encoder reranking (Solr chunks unchanged)
 BYOK_RAG_RERANK_BOOST: Final[float] = 1.2
 
+# Default minimum raw similarity per BYOK store
+DEFAULT_BYOK_RAG_RELEVANCE_CUTOFF_SCORE: Final[float] = 0.3
+
 # Solr OKP constants
 SOLR_VECTOR_SEARCH_DEFAULT_K: Final[int] = 5
 SOLR_VECTOR_SEARCH_DEFAULT_SCORE_THRESHOLD: Final[float] = 0.3

src/models/config.py

@@ -1876,6 +1876,14 @@ class ByokRag(ConfigurationBase):
         "Values > 1 boost this store's results; values < 1 reduce them.",
     )
 
+    relevance_cutoff_score: float = Field(
+        constants.DEFAULT_BYOK_RAG_RELEVANCE_CUTOFF_SCORE,
+        gt=0,
+        title="Relevance cutoff score",
+        description="Minimum raw similarity score to consider a result relevant. "
+        "Results with a similarity score below this threshold are not returned.",
+    )
+
     host: Optional[str] = Field(
         default=None,
         title="PostgreSQL host",
@@ -1930,7 +1938,6 @@ def validate_rag_type_fields(self) -> Self:
                     object.__setattr__(self, field_name, default_value)
         return self
 
-
 class QuotaLimiterConfiguration(ConfigurationBase):
     """Configuration for one quota limiter.
 

src/utils/vector_search.py

@@ -250,6 +250,7 @@ async def _query_store_for_byok_rag(
     vector_store_id: str,
     query: str,
     weight: float,
+    score_threshold: float,
     max_chunks: int = constants.BYOK_RAG_MAX_CHUNKS,
 ) -> list[dict[str, Any]]:
     """Query a single vector store for BYOK RAG.
@@ -260,6 +261,7 @@ async def _query_store_for_byok_rag(
         query: Search query string
         weight: Score multiplier to apply
         max_chunks: Maximum number of chunks to request from this store.
+        score_threshold: Minimum raw similarity score (``relevance_cutoff_score``)
 
     Returns:
         List of weighted result dictionaries, or empty list on error
@@ -271,6 +273,7 @@ async def _query_store_for_byok_rag(
             params={
                 "max_chunks": max_chunks,
                 "mode": "vector",
+                "score_threshold": score_threshold,
             },
         )
         return _extract_byok_rag_chunks(search_response, vector_store_id, weight)
@@ -496,8 +499,9 @@ async def _fetch_byok_rag(  # pylint: disable=too-many-locals
         return rag_chunks, referenced_documents
 
     try:
-        # Get score multiplier and rag_id mappings
+        # Get per-store mappings from configuration
         score_multiplier_mapping = configuration.score_multiplier_mapping
+        relevance_cutoff_mapping = configuration.relevance_cutoff_mapping
         rag_id_mapping = configuration.rag_id_mapping
 
         # Query all vector stores in parallel
@@ -508,6 +512,10 @@ async def _fetch_byok_rag(  # pylint: disable=too-many-locals
                     vector_store_id,
                     query,
                     score_multiplier_mapping.get(vector_store_id, 1.0),
+                    relevance_cutoff_mapping.get(
+                        vector_store_id,
+                        constants.DEFAULT_BYOK_RAG_RELEVANCE_CUTOFF_SCORE,
+                    ),
                     max_chunks=limit,
                 )
                 for vector_store_id in vector_store_ids_to_query

tests/unit/models/config/test_byok_rag.py

@@ -4,6 +4,7 @@
 from pydantic import ValidationError
 
 from constants import (
+    DEFAULT_BYOK_RAG_RELEVANCE_CUTOFF_SCORE,
     DEFAULT_EMBEDDING_DIMENSION,
     DEFAULT_EMBEDDING_MODEL,
     DEFAULT_RAG_TYPE,
@@ -35,6 +36,7 @@ def test_byok_rag_configuration_default_values() -> None:
     assert byok_rag.vector_db_id == "vector_db_id"
     assert byok_rag.db_path == "tests/configuration/rag.txt"
     assert byok_rag.score_multiplier == DEFAULT_SCORE_MULTIPLIER
+    assert byok_rag.relevance_cutoff_score == DEFAULT_BYOK_RAG_RELEVANCE_CUTOFF_SCORE
 
 
 def test_byok_rag_configuration_nondefault_values() -> None:
@@ -54,6 +56,7 @@ def test_byok_rag_configuration_nondefault_values() -> None:
         vector_db_id="vector_db_id",
         db_path="tests/configuration/rag.txt",
         score_multiplier=1.0,
+        relevance_cutoff_score=0.72,
     )
     assert byok_rag is not None
     assert byok_rag.rag_id == "rag_id"
@@ -62,6 +65,7 @@ def test_byok_rag_configuration_nondefault_values() -> None:
     assert byok_rag.embedding_dimension == 1024
     assert byok_rag.vector_db_id == "vector_db_id"
     assert byok_rag.db_path == "tests/configuration/rag.txt"
+    assert byok_rag.relevance_cutoff_score == 0.72
 
 
 def test_byok_rag_configuration_wrong_dimension() -> None:
@@ -187,7 +191,10 @@ def test_byok_rag_configuration_custom_score_multiplier() -> None:
     assert byok_rag.score_multiplier == 2.5
 
 
-def test_byok_rag_configuration_score_multiplier_must_be_positive() -> None:
+@pytest.mark.parametrize("bad_multiplier", [0.0, -0.5])
+def test_byok_rag_configuration_score_multiplier_must_be_positive(
+    bad_multiplier: float,
+) -> None:
     """Test that score_multiplier must be greater than 0."""
     with pytest.raises(ValidationError, match="greater than 0"):
         _ = ByokRag(
@@ -197,7 +204,25 @@ def test_byok_rag_configuration_score_multiplier_must_be_positive() -> None:
             embedding_model="embedding_model",
             embedding_dimension=1024,
             db_path="tests/configuration/rag.txt",
-            score_multiplier=0.0,
+            score_multiplier=bad_multiplier,
+        )
+
+
+@pytest.mark.parametrize("bad_cutoff", [0.0, -0.5])
+def test_byok_rag_configuration_relevance_cutoff_must_be_positive(
+    bad_cutoff: float,
+) -> None:
+    """Test that relevance_cutoff_score must be greater than 0."""
+    with pytest.raises(ValidationError, match="greater than 0"):
+        _ = ByokRag(
+            rag_id="rag_id",
+            rag_type="rag_type",
+            vector_db_id="vector_db_id",
+            embedding_model="embedding_model",
+            embedding_dimension=1024,
+            db_path="tests/configuration/rag.txt",
+            score_multiplier=1.0,
+            relevance_cutoff_score=bad_cutoff,
         )
 
 

tests/unit/models/config/test_dump_configuration.py

@@ -1250,6 +1250,7 @@ def test_dump_configuration_byok(tmp_path: Path) -> None:
                     "rag_type": "inline::faiss",
                     "vector_db_id": "vector_db_id",
                     "score_multiplier": 1.0,
+                    "relevance_cutoff_score": constants.DEFAULT_BYOK_RAG_RELEVANCE_CUTOFF_SCORE,                    
                     "host": None,
                     "port": None,
                     "db": None,

tests/unit/test_configuration.py

@@ -1195,6 +1195,86 @@ def test_score_multiplier_mapping_not_loaded() -> None:
         _ = cfg.score_multiplier_mapping
 
 
+def test_relevance_cutoff_mapping_empty_when_no_byok(minimal_config: AppConfig) -> None:
+    """Test that relevance_cutoff_mapping returns empty dict when no BYOK RAG configured."""
+    assert minimal_config.relevance_cutoff_mapping == {}
+
+
+def test_relevance_cutoff_mapping_with_byok_defaults(tmp_path: Path) -> None:
+    """Test that relevance_cutoff_mapping uses default cutoff when not specified."""
+    db_file = tmp_path / "test.db"
+    db_file.touch()
+    cfg = AppConfig()
+    cfg.init_from_dict(
+        {
+            "name": "test",
+            "service": {"host": "localhost", "port": 8080},
+            "llama_stack": {
+                "api_key": "k",
+                "url": "http://test.com:1234",
+                "use_as_library_client": False,
+            },
+            "user_data_collection": {},
+            "authentication": {"module": "noop"},
+            "byok_rag": [
+                {
+                    "rag_id": "my-kb",
+                    "vector_db_id": "vs-001",
+                    "db_path": str(db_file),
+                },
+            ],
+        }
+    )
+    assert cfg.relevance_cutoff_mapping == {
+        "vs-001": constants.DEFAULT_BYOK_RAG_RELEVANCE_CUTOFF_SCORE,
+    }
+
+
+def test_relevance_cutoff_mapping_with_custom_values(tmp_path: Path) -> None:
+    """Test that relevance_cutoff_mapping builds correct mapping with custom values."""
+    db_file1 = tmp_path / "test1.db"
+    db_file1.touch()
+    db_file2 = tmp_path / "test2.db"
+    db_file2.touch()
+    cfg = AppConfig()
+    cfg.init_from_dict(
+        {
+            "name": "test",
+            "service": {"host": "localhost", "port": 8080},
+            "llama_stack": {
+                "api_key": "k",
+                "url": "http://test.com:1234",
+                "use_as_library_client": False,
+            },
+            "user_data_collection": {},
+            "authentication": {"module": "noop"},
+            "byok_rag": [
+                {
+                    "rag_id": "kb1",
+                    "vector_db_id": "vs-001",
+                    "db_path": str(db_file1),
+                    "relevance_cutoff_score": 0.55,
+                },
+                {
+                    "rag_id": "kb2",
+                    "vector_db_id": "vs-002",
+                    "db_path": str(db_file2),
+                    "relevance_cutoff_score": 0.1,
+                },
+            ],
+        }
+    )
+    assert cfg.relevance_cutoff_mapping == {"vs-001": 0.55, "vs-002": 0.1}
+
+
+def test_relevance_cutoff_mapping_not_loaded() -> None:
+    """Test that relevance_cutoff_mapping raises when config not loaded."""
+    cfg = AppConfig()
+    cfg._configuration = None
+    with pytest.raises(LogicError):
+        _ = cfg.relevance_cutoff_mapping
+
+
 wrong_configurations = [
     {
         "name": "Colin Adams",