Merge pull request #1407 from are-ces/LCORE-1500-inline-rag-config-filtering

LCORE-1500: Fix inline RAG triggered by unconfigured vector_store_ids

Author: tisnik

src/utils/vector_search.py

@@ -352,12 +352,17 @@ async def _fetch_byok_rag(
     referenced_documents: list[ReferencedDocument] = []
 
     # Determine which BYOK vector stores to query for inline RAG.
-    # Per-request override takes precedence; otherwise use config-based inline list.
-    rag_ids_to_query = (
-        configuration.configuration.rag.inline
-        if vector_store_ids is None
-        else vector_store_ids
-    )
+    # Config is the source of truth: only rag_ids registered in rag.inline are eligible.
+    # Per-request IDs are intersected with the config to prevent triggering inline RAG
+    # for stores not explicitly configured for inline use.
+    if vector_store_ids is None:
+        rag_ids_to_query = configuration.configuration.rag.inline
+    else:
+        rag_ids_to_query = [
+            v
+            for v in vector_store_ids
+            if v in set(configuration.configuration.rag.inline)
+        ]
 
     # Translate user-facing rag_ids to llama-stack ids
     vector_store_ids_to_query: list[str] = resolve_vector_store_ids(

tests/integration/endpoints/test_query_byok_integration.py

@@ -404,16 +404,14 @@ async def test_query_byok_inline_rag_with_request_vector_store_ids(
     test_request: Request,
     test_auth: AuthTuple,
 ) -> None:
-    """Test that per-request vector_store_ids override config-based inline RAG.
+    """Test that per-request vector_store_ids not in rag.inline are filtered out.
 
     Config has rag.inline = ["source-a"] (resolves to vs-source-a).
-    Request passes vector_store_ids = ["vs-source-b"].
-    Only vs-source-b should be queried, proving the override works.
-    (passing vector_store_ids overrides config)
+    Request passes vector_store_ids = ["source-b"] which is NOT in rag.inline.
+    No inline RAG should be triggered because config is the source of truth.
 
     Verifies:
-    - vector_io.query is called with the request-specified store, not config
-    - The config-based store is NOT queried
+    - vector_io.query is NOT called (source-b is not in rag.inline)
     """
     entry_a = mocker.MagicMock()
     entry_a.rag_id = "source-a"
@@ -441,10 +439,10 @@ async def test_query_byok_inline_rag_with_request_vector_store_ids(
 
     mock_holder_class.return_value.get_client.return_value = mock_client
 
-    # Override: request specifies vs-source-b, not the config's vs-source-a
+    # Request specifies source-b which is NOT in rag.inline config
     query_request = QueryRequest(
         query="What is OpenShift?",
-        vector_store_ids=["vs-source-b"],
+        vector_store_ids=["source-b"],
     )
 
     await query_endpoint_handler(
@@ -454,12 +452,8 @@ async def test_query_byok_inline_rag_with_request_vector_store_ids(
         mcp_headers={},
     )
 
-    # Verify only vs-source-b was queried (not the config's vs-source-a)
-    assert mock_client.vector_io.query.call_count == 1
-    # call_args.kwargs holds the keyword arguments of the most recent call to vector_io.query.
-    # e.g. "vector_store_id" is the store queried, "query" is the search text.
-    call_kwargs = mock_client.vector_io.query.call_args.kwargs
-    assert call_kwargs["vector_store_id"] == "vs-source-b"
+    # source-b is not in rag.inline, so no inline RAG should be triggered
+    assert mock_client.vector_io.query.call_count == 0
 
 
 @pytest.mark.asyncio
@@ -506,10 +500,10 @@ async def test_query_byok_request_vector_store_ids_filters_configured_stores(
 
     mock_holder_class.return_value.get_client.return_value = mock_client
 
-    # Request narrows down to only vs-source-a
+    # Request narrows down to only source-a (using rag_id, not vector_db_id)
     query_request = QueryRequest(
         query="What is OpenShift?",
-        vector_store_ids=["vs-source-a"],
+        vector_store_ids=["source-a"],
     )
 
     response = await query_endpoint_handler(

tests/integration/endpoints/test_streaming_query_byok_integration.py

@@ -324,16 +324,14 @@ async def test_streaming_query_byok_inline_rag_with_request_vector_store_ids(
     test_request: Request,
     test_auth: AuthTuple,
 ) -> None:
-    """Test that per-request vector_store_ids override config for streaming query.
+    """Test that per-request vector_store_ids not in rag.inline are filtered out.
 
     Config has rag.inline = ["source-a"] (resolves to vs-source-a).
-    Request passes vector_store_ids = ["vs-source-b"].
-    Only vs-source-b should be queried, proving the override works.
-    (passing vector_store_ids overrides config)
+    Request passes vector_store_ids = ["source-b"] which is NOT in rag.inline.
+    No inline RAG should be triggered because config is the source of truth.
 
     Verifies:
-    - vector_io.query is called with the request-specified store, not config
-    - The config-based store is NOT queried
+    - vector_io.query is NOT called (source-b is not in rag.inline)
     """
     entry_a = mocker.MagicMock()
     entry_a.rag_id = "source-a"
@@ -363,10 +361,10 @@ async def test_streaming_query_byok_inline_rag_with_request_vector_store_ids(
 
     mock_holder_class.return_value.get_client.return_value = mock_client
 
-    # Override: request specifies vs-source-b, not the config's vs-source-a
+    # Request specifies source-b which is NOT in rag.inline config
     query_request = QueryRequest(
         query="What is OpenShift?",
-        vector_store_ids=["vs-source-b"],
+        vector_store_ids=["source-b"],
     )
 
     response = await streaming_query_endpoint_handler(
@@ -378,12 +376,8 @@ async def test_streaming_query_byok_inline_rag_with_request_vector_store_ids(
 
     assert isinstance(response, StreamingResponse)
 
-    # Verify only vs-source-b was queried (not the config's vs-source-a)
-    assert mock_client.vector_io.query.call_count == 1
-    # call_args.kwargs holds the keyword arguments of the most recent call to vector_io.query.
-    # e.g. "vector_store_id" is the store queried, "query" is the search text.
-    call_kwargs = mock_client.vector_io.query.call_args.kwargs
-    assert call_kwargs["vector_store_id"] == "vs-source-b"
+    # source-b is not in rag.inline, so no inline RAG should be triggered
+    assert mock_client.vector_io.query.call_count == 0
 
 
 @pytest.mark.asyncio
@@ -433,7 +427,7 @@ async def test_streaming_query_byok_request_vector_store_ids_filters_configured_
 
     query_request = QueryRequest(
         query="What is OpenShift?",
-        vector_store_ids=["vs-source-a"],
+        vector_store_ids=["source-a"],
     )
 
     response = await streaming_query_endpoint_handler(

tests/unit/utils/test_vector_search.py

@@ -440,6 +440,7 @@ async def test_user_facing_ids_translated_to_internal_ids(
         byok_rag_mock.rag_id = "my-kb"
         byok_rag_mock.vector_db_id = "vs-internal-001"
         config_mock.configuration.byok_rag = [byok_rag_mock]
+        config_mock.configuration.rag.inline = ["my-kb"]
         config_mock.score_multiplier_mapping = {"vs-internal-001": 1.0}
         config_mock.rag_id_mapping = {"vs-internal-001": "my-kb"}
         mocker.patch("utils.vector_search.configuration", config_mock)
@@ -479,6 +480,7 @@ async def test_multiple_user_facing_ids_each_translated(
         byok_rag_2.rag_id = "kb-part2"
         byok_rag_2.vector_db_id = "vs-bbb-222"
         config_mock.configuration.byok_rag = [byok_rag_1, byok_rag_2]
+        config_mock.configuration.rag.inline = ["kb-part1", "kb-part2"]
         config_mock.score_multiplier_mapping = {"vs-aaa-111": 1.0, "vs-bbb-222": 1.0}
         config_mock.rag_id_mapping = {
             "vs-aaa-111": "kb-part1",
@@ -513,6 +515,46 @@ async def test_multiple_user_facing_ids_each_translated(
         assert "kb-part1" not in call_args
         assert "kb-part2" not in call_args
 
+    @pytest.mark.asyncio
+    async def test_no_inline_rag_configured_skips_byok(
+        self, mocker: MockerFixture
+    ) -> None:
+        """Test that BYOK inline RAG is skipped when rag.inline is empty."""
+        config_mock = mocker.Mock(spec=AppConfig)
+        config_mock.configuration.rag.inline = []
+        config_mock.configuration.byok_rag = []
+        mocker.patch("utils.vector_search.configuration", config_mock)
+
+        client_mock = mocker.AsyncMock()
+
+        rag_chunks, referenced_docs = await _fetch_byok_rag(
+            client_mock, "test query", vector_store_ids=["some-id"]
+        )
+
+        assert rag_chunks == []
+        assert referenced_docs == []
+        client_mock.vector_io.query.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_request_id_not_in_inline_config_skips_byok(
+        self, mocker: MockerFixture
+    ) -> None:
+        """Test that a request vector_store_id not registered in rag.inline is filtered out."""
+        config_mock = mocker.Mock(spec=AppConfig)
+        config_mock.configuration.rag.inline = ["registered-id"]
+        config_mock.configuration.byok_rag = []
+        mocker.patch("utils.vector_search.configuration", config_mock)
+
+        client_mock = mocker.AsyncMock()
+
+        rag_chunks, referenced_docs = await _fetch_byok_rag(
+            client_mock, "test query", vector_store_ids=["unregistered-id"]
+        )
+
+        assert rag_chunks == []
+        assert referenced_docs == []
+        client_mock.vector_io.query.assert_not_called()
+
 
 class TestFetchSolrRag:
     """Tests for _fetch_solr_rag async function."""