LCORE-1426: address code review feedback

- Add SUPPORTED_RAG_BACKENDS constant and field validator on
  RagStore.backend to reject unsupported backends at config load time
- Add model validator on RagConfiguration to reject retrieval source
  IDs not declared in byok.stores
- Add assert that DEFAULT_RAG_BACKEND has a BACKEND_TO_LLAMA_STACK_PROVIDER
  mapping
- Remove llama-stack references from docs, use run.yaml instead
- Add backward compatibility note to RAG guide
- Update tests for new validators

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: are-ces

docs/byok_guide.md

@@ -317,7 +317,7 @@ byok_rag:
 
 > [!NOTE]
 > pgvector is not yet supported via `byok_rag` in `lightspeed-stack.yaml` (see [LCORE-2437](https://redhat.atlassian.net/browse/LCORE-2437)).
-> It must be configured directly in the Llama Stack configuration file.
+> It must be configured directly in the `run.yaml` configuration file.
 
 ```yaml
 vector_io:
@@ -371,11 +371,11 @@ rag:
 
 ### Example 2: Multiple Knowledge Sources with pgvector
 
-A configuration combining a local FAISS store (via `byok_rag`) with a remote pgvector store (configured directly in the Llama Stack configuration file):
+A configuration combining a local FAISS store (via `byok_rag`) with a remote pgvector store (configured directly in the `run.yaml` configuration file):
 
 > [!NOTE]
 > pgvector is not yet supported via `byok_rag` in `lightspeed-stack.yaml` (see [LCORE-2437](https://redhat.atlassian.net/browse/LCORE-2437)).
-> The pgvector provider must be configured directly in the Llama Stack configuration file.
+> The pgvector provider must be configured directly in the `run.yaml` configuration file.
 
 **`lightspeed-stack.yaml`** — FAISS store and RAG strategy:
 
@@ -402,7 +402,7 @@ rag:
     - local-docs
 ```
 
-**Llama Stack configuration file** — pgvector provider:
+**`run.yaml` configuration file** — pgvector provider:
 
 ```yaml
 vector_io:

docs/rag_guide.md

@@ -36,7 +36,7 @@ Both strategies can be enabled independently via the `rag` section of `lightspee
 
 > [!NOTE]
 > **Backward compatibility:** if neither `retrieval.inline.sources` nor `retrieval.tool.sources` is
-> configured, all BYOK vector stores registered in llama-stack are automatically exposed as
+> configured, all registered vector stores (BYOK and OKP) are automatically exposed as
 > Tool RAG (`file_search`). Inline RAG is **not** enabled in this fallback — only Tool RAG.
 
 ### Inline RAG chunk flow
@@ -122,7 +122,7 @@ This example shows how to configure a remote PostgreSQL database with the [pgvec
 
 > [!NOTE]
 > pgvector is not yet supported via `byok_rag` in `lightspeed-stack.yaml` (see [LCORE-2437](https://redhat.atlassian.net/browse/LCORE-2437)).
-> It must be configured directly in the Llama Stack configuration file.
+> It must be configured directly in the `run.yaml` configuration file.
 
 > You will need to install PostgreSQL with a matching version to pgvector, then log in with `psql` and enable the extension with:
 > ```sql

src/constants.py

@@ -169,6 +169,9 @@
 CACHE_TYPE_NOOP: Final[str] = "noop"
 
 # BYOK RAG
+# Backends that have enrichment support in llama_stack_configuration.py
+SUPPORTED_RAG_BACKENDS: Final[frozenset[str]] = frozenset({"faiss"})
+
 # Default RAG backend for bring-your-own-knowledge RAG configurations
 DEFAULT_RAG_BACKEND: Final[str] = "faiss"
 

src/llama_stack_configuration.py

@@ -22,6 +22,11 @@
     # "pgvector": "remote::pgvector",  # TODO(are-ces): add enrichment support
 }
 
+assert constants.DEFAULT_RAG_BACKEND in BACKEND_TO_LLAMA_STACK_PROVIDER, (
+    f"DEFAULT_RAG_BACKEND '{constants.DEFAULT_RAG_BACKEND}' has no entry in "
+    f"BACKEND_TO_LLAMA_STACK_PROVIDER — add a mapping before changing the default."
+)
+
 
 class YamlDumper(yaml.Dumper):  # pylint: disable=too-many-ancestors
     """Custom YAML dumper with proper indentation levels."""

src/models/config.py

@@ -1779,6 +1779,17 @@ class RagStore(ConfigurationBase):
         description="Storage backend type (e.g. 'faiss').",
     )
 
+    @field_validator("backend")
+    @classmethod
+    def validate_backend(cls, value: str) -> str:
+        """Reject unsupported backend values at config load time."""
+        if value not in constants.SUPPORTED_RAG_BACKENDS:
+            raise ValueError(
+                f"Unsupported RAG backend '{value}'. "
+                f"Supported backends: {sorted(constants.SUPPORTED_RAG_BACKENDS)}"
+            )
+        return value
+
     embedding_model: str = Field(
         constants.DEFAULT_EMBEDDING_MODEL,
         min_length=1,
@@ -2052,6 +2063,24 @@ class RagConfiguration(ConfigurationBase):
         description="Inline and tool retrieval strategy settings.",
     )
 
+    @model_validator(mode="after")
+    def validate_retrieval_sources(self) -> Self:
+        """Reject retrieval source IDs not declared in byok.stores or OKP."""
+        known_ids = {store.rag_id for store in self.byok.stores}
+        known_ids.add(constants.OKP_RAG_ID)
+
+        for strategy_name in ("inline", "tool"):
+            strategy = getattr(self.retrieval, strategy_name)
+            unknown = set(strategy.sources) - known_ids
+            if unknown:
+                raise ValueError(
+                    f"retrieval.{strategy_name}.sources contains unknown RAG IDs: "
+                    f"{sorted(unknown)}. "
+                    f"Declared IDs: {sorted(known_ids)}"
+                )
+
+        return self
+
 
 class RerankerConfiguration(ConfigurationBase):
     """Reranker configuration for RAG chunk reranking."""

tests/unit/models/config/test_byok_rag.py

@@ -48,7 +48,7 @@ def test_rag_store_configuration_nondefault_values() -> None:
     """
     rag_store = RagStore(
         rag_id="rag_id",
-        backend="custom_backend",
+        backend="faiss",
         embedding_model="embedding_model",
         embedding_dimension=1024,
         vector_db_id="vector_db_id",
@@ -57,7 +57,7 @@ def test_rag_store_configuration_nondefault_values() -> None:
     )
     assert rag_store is not None
     assert rag_store.rag_id == "rag_id"
-    assert rag_store.backend == "custom_backend"
+    assert rag_store.backend == "faiss"
     assert rag_store.embedding_model == "embedding_model"
     assert rag_store.embedding_dimension == 1024
     assert rag_store.vector_db_id == "vector_db_id"
@@ -75,7 +75,7 @@ def test_rag_store_configuration_wrong_dimension() -> None:
     with pytest.raises(ValidationError, match="should be greater than 0"):
         _ = RagStore(
             rag_id="rag_id",
-            backend="custom_backend",
+            backend="faiss",
             embedding_model="embedding_model",
             embedding_dimension=-1024,
             vector_db_id="vector_db_id",
@@ -97,7 +97,7 @@ def test_rag_store_configuration_empty_rag_id() -> None:
     ):
         _ = RagStore(
             rag_id="",
-            backend="custom_backend",
+            backend="faiss",
             embedding_model="embedding_model",
             embedding_dimension=1024,
             vector_db_id="vector_db_id",
@@ -129,6 +129,19 @@ def test_rag_store_configuration_empty_backend() -> None:
         )
 
 
+def test_rag_store_configuration_unsupported_backend() -> None:
+    """Test that unsupported backend values are rejected."""
+    with pytest.raises(ValidationError, match="Unsupported RAG backend"):
+        _ = RagStore(
+            rag_id="rag_id",
+            backend="unsupported",
+            embedding_model="embedding_model",
+            embedding_dimension=1024,
+            vector_db_id="vector_db_id",
+            db_path="tests/configuration/rag.txt",
+        )
+
+
 def test_rag_store_configuration_empty_embedding_model() -> None:
     """Test the RagStore constructor.
 
@@ -142,7 +155,7 @@ def test_rag_store_configuration_empty_embedding_model() -> None:
     ):
         _ = RagStore(
             rag_id="rag_id",
-            backend="custom_backend",
+            backend="faiss",
             embedding_model="",
             embedding_dimension=1024,
             vector_db_id="vector_db_id",
@@ -164,7 +177,7 @@ def test_rag_store_configuration_empty_vector_db_id() -> None:
     ):
         _ = RagStore(
             rag_id="rag_id",
-            backend="custom_backend",
+            backend="faiss",
             embedding_model="embedding_model",
             embedding_dimension=1024,
             vector_db_id="",
@@ -177,7 +190,7 @@ def test_rag_store_configuration_custom_score_multiplier() -> None:
     """Test RagStore with custom score_multiplier."""
     rag_store = RagStore(
         rag_id="rag_id",
-        backend="custom_backend",
+        backend="faiss",
         vector_db_id="vector_db_id",
         embedding_model="embedding_model",
         embedding_dimension=1024,
@@ -192,7 +205,7 @@ def test_rag_store_configuration_score_multiplier_must_be_positive() -> None:
     with pytest.raises(ValidationError, match="greater than 0"):
         _ = RagStore(
             rag_id="rag_id",
-            backend="custom_backend",
+            backend="faiss",
             vector_db_id="vector_db_id",
             embedding_model="embedding_model",
             embedding_dimension=1024,

tests/unit/models/config/test_rag_configuration.py

@@ -97,7 +97,12 @@ def test_default_values(self) -> None:
 
     def test_inline_with_byok_ids(self) -> None:
         """Test inline sources with BYOK rag IDs."""
+        stores = [
+            RagStore(rag_id="store-1", vector_db_id="vs_1", db_path="/tmp/s1.db"),
+            RagStore(rag_id="store-2", vector_db_id="vs_2", db_path="/tmp/s2.db"),
+        ]
         config = RagConfiguration(
+            byok=ByokConfiguration(stores=stores),
             retrieval=RetrievalConfiguration(
                 inline=RetrievalStrategyConfiguration(sources=["store-1", "store-2"]),
             ),
@@ -107,7 +112,9 @@ def test_inline_with_byok_ids(self) -> None:
 
     def test_inline_with_okp_rag(self) -> None:
         """Test inline sources including the special OKP ID."""
+        store = RagStore(rag_id="store-1", vector_db_id="vs_1", db_path="/tmp/s1.db")
         config = RagConfiguration(
+            byok=ByokConfiguration(stores=[store]),
             retrieval=RetrievalConfiguration(
                 inline=RetrievalStrategyConfiguration(
                     sources=[constants.OKP_RAG_ID, "store-1"]
@@ -119,7 +126,9 @@ def test_inline_with_okp_rag(self) -> None:
 
     def test_tool_with_okp_rag_and_byok(self) -> None:
         """Test tool sources with OKP and BYOK IDs."""
+        store = RagStore(rag_id="store-1", vector_db_id="vs_1", db_path="/tmp/s1.db")
         config = RagConfiguration(
+            byok=ByokConfiguration(stores=[store]),
             retrieval=RetrievalConfiguration(
                 inline=RetrievalStrategyConfiguration(sources=["store-1"]),
                 tool=RetrievalStrategyConfiguration(
@@ -144,6 +153,28 @@ def test_tool_default_is_empty_list(self) -> None:
         config = RagConfiguration()
         assert config.retrieval.tool.sources == []
 
+    def test_unknown_inline_source_rejected(self) -> None:
+        """Test that inline sources referencing undeclared rag_ids are rejected."""
+        store = RagStore(rag_id="store-1", vector_db_id="vs_1", db_path="/tmp/s1.db")
+        with pytest.raises(ValidationError, match="unknown RAG IDs"):
+            RagConfiguration(
+                byok=ByokConfiguration(stores=[store]),
+                retrieval=RetrievalConfiguration(
+                    inline=RetrievalStrategyConfiguration(
+                        sources=["store-1", "nonexistent"]
+                    ),
+                ),
+            )
+
+    def test_unknown_tool_source_rejected(self) -> None:
+        """Test that tool sources referencing undeclared rag_ids are rejected."""
+        with pytest.raises(ValidationError, match="unknown RAG IDs"):
+            RagConfiguration(
+                retrieval=RetrievalConfiguration(
+                    tool=RetrievalStrategyConfiguration(sources=["missing-store"]),
+                ),
+            )
+
     def test_no_unknown_fields_allowed(self) -> None:
         """Test that RagConfiguration rejects unknown fields."""
         with pytest.raises(ValidationError, match="Extra inputs are not permitted"):