fix: add default model availability check to readiness probe

major · major · commit 7d37c15263ba · 2026-04-29T14:48:58.000-05:00
The /readiness endpoint only verified provider health but did not
check that the configured default model was registered in the Llama
Stack model registry. This allowed pods where model registration
failed during startup to pass readiness and serve 404s on every
inference request.

Add check_default_model_available() that verifies the configured
default model exists in client.models.list(). When the model is
missing, /readiness returns 503 so Kubernetes removes the pod from
the service load balancer.

Ref: RSPEED-2959
Signed-off-by: Major Hayden &lt;major@redhat.com&gt;
diff --git a/src/app/endpoints/health.py b/src/app/endpoints/health.py
@@ -9,12 +9,13 @@
 from typing import Annotated, Any
 
 from fastapi import APIRouter, Depends, Response, status
-from llama_stack_client import APIConnectionError
+from llama_stack_client import APIConnectionError, APIStatusError
 
 from authentication import get_auth_dependency
 from authentication.interface import AuthTuple
 from authorization.middleware import authorize
 from client import AsyncLlamaStackClientHolder
+from configuration import configuration
 from log import get_logger
 from models.config import Action
 from models.responses import (
@@ -95,6 +96,53 @@ async def get_providers_health_statuses() -> list[ProviderHealthStatus]:
         ]
 
 
+async def check_default_model_available() -> tuple[bool, str]:
+    """Check that the configured default model is registered in the model registry.
+
+    Verifies the default model from configuration can be found in the Llama
+    Stack client's model list. This catches cases where a pod started
+    successfully and providers report healthy, but model registration failed
+    during initialization.
+
+    Returns:
+        A tuple of (available, reason) where available is True if the default
+        model was found or no default model is configured, and reason describes
+        the outcome.
+    """
+    if configuration.inference is None:
+        return True, "No inference configuration"
+
+    default_model = configuration.inference.default_model
+    default_provider = configuration.inference.default_provider
+
+    if not default_model or not default_provider:
+        return True, "No default model configured"
+
+    expected_model_id = f"{default_provider}/{default_model}"
+
+    try:
+        client = AsyncLlamaStackClientHolder().get_client()
+        models = await client.models.list()
+
+        for model in models:
+            if model.id == expected_model_id:
+                return True, f"Default model {expected_model_id} is available"
+
+        registered_ids = [m.id for m in models]
+        logger.error(
+            "Default model %s not found in registry. Registered models: %s",
+            expected_model_id,
+            registered_ids,
+        )
+        return False, (f"Default model {expected_model_id} not found in model registry")
+    except APIConnectionError as e:
+        logger.error("Failed to check model availability: %s", e)
+        return False, f"Failed to check model availability: {e!s}"
+    except APIStatusError as e:
+        logger.error("API error checking model availability: %s", e)
+        return False, f"API error checking model availability: {e!s}"
+
+
 @router.get("/readiness", responses=get_readiness_responses)
 @authorize(Action.INFO)
 async def readiness_probe_get_method(
@@ -134,11 +182,21 @@ async def readiness_probe_get_method(
         unhealthy_provider_names = [p.provider_id for p in unhealthy_providers]
         reason = f"Providers not healthy: {', '.join(unhealthy_provider_names)}"
         response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
-    else:
-        ready = True
-        reason = "All providers are healthy"
+        return ReadinessResponse(
+            ready=ready, reason=reason, providers=unhealthy_providers
+        )
+
+    # Check that the default model is registered in the model registry
+    model_available, model_reason = await check_default_model_available()
+    if not model_available:
+        response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
+        return ReadinessResponse(
+            ready=False, reason=model_reason, providers=unhealthy_providers
+        )
 
-    return ReadinessResponse(ready=ready, reason=reason, providers=unhealthy_providers)
+    return ReadinessResponse(
+        ready=True, reason="All providers are healthy", providers=unhealthy_providers
+    )
 
 
 @router.get("/liveness", responses=get_liveness_responses)
diff --git a/tests/unit/app/endpoints/test_health.py b/tests/unit/app/endpoints/test_health.py
@@ -1,11 +1,12 @@
 """Unit tests for the /health REST API endpoint."""
 
 import pytest
-from llama_stack_client import APIConnectionError
+from llama_stack_client import APIConnectionError, APIStatusError
 from pytest_mock import MockerFixture
 
 from app.endpoints.health import (
     HealthStatus,
+    check_default_model_available,
     get_providers_health_statuses,
     liveness_probe_get_method,
     readiness_probe_get_method,
@@ -207,3 +208,189 @@ async def test_get_providers_health_statuses_connection_error(
         assert (
             result[0].message == "Failed to initialize health check: Connection error."
         )
+
+
+class TestCheckDefaultModelAvailable:
+    """Test cases for the check_default_model_available function."""
+
+    @pytest.mark.asyncio
+    async def test_model_available(self, mocker: MockerFixture) -> None:
+        """Test returns True when the default model is found in the registry."""
+        mock_config = mocker.patch("app.endpoints.health.configuration")
+        mock_config.inference.default_model = (
+            "publishers/google/models/gemini-2.5-flash"
+        )
+        mock_config.inference.default_provider = "google-vertex"
+
+        mock_lsc = mocker.patch("app.endpoints.health.AsyncLlamaStackClientHolder")
+        mock_client = mocker.AsyncMock()
+        mock_lsc.return_value.get_client.return_value = mock_client
+
+        mock_model = mocker.Mock()
+        mock_model.id = "google-vertex/publishers/google/models/gemini-2.5-flash"
+        mock_client.models.list.return_value = [mock_model]
+
+        available, reason = await check_default_model_available()
+
+        assert available is True
+        assert "is available" in reason
+
+    @pytest.mark.asyncio
+    async def test_model_not_found(self, mocker: MockerFixture) -> None:
+        """Test returns False when the default model is missing from the registry."""
+        mock_config = mocker.patch("app.endpoints.health.configuration")
+        mock_config.inference.default_model = (
+            "publishers/google/models/gemini-2.5-flash"
+        )
+        mock_config.inference.default_provider = "google-vertex"
+
+        mock_lsc = mocker.patch("app.endpoints.health.AsyncLlamaStackClientHolder")
+        mock_client = mocker.AsyncMock()
+        mock_lsc.return_value.get_client.return_value = mock_client
+
+        mock_model = mocker.Mock()
+        mock_model.id = "some-other-provider/some-other-model"
+        mock_client.models.list.return_value = [mock_model]
+
+        available, reason = await check_default_model_available()
+
+        assert available is False
+        assert "not found in model registry" in reason
+
+    @pytest.mark.asyncio
+    async def test_no_inference_config(self, mocker: MockerFixture) -> None:
+        """Test returns True when no inference configuration exists."""
+        mock_config = mocker.patch("app.endpoints.health.configuration")
+        mock_config.inference = None
+
+        available, reason = await check_default_model_available()
+
+        assert available is True
+        assert reason == "No inference configuration"
+
+    @pytest.mark.asyncio
+    async def test_no_default_model_configured(self, mocker: MockerFixture) -> None:
+        """Test returns True when no default model is configured."""
+        mock_config = mocker.patch("app.endpoints.health.configuration")
+        mock_config.inference.default_model = None
+        mock_config.inference.default_provider = None
+
+        available, reason = await check_default_model_available()
+
+        assert available is True
+        assert reason == "No default model configured"
+
+    @pytest.mark.asyncio
+    async def test_connection_error(self, mocker: MockerFixture) -> None:
+        """Test returns False when model list call fails with connection error."""
+        mock_config = mocker.patch("app.endpoints.health.configuration")
+        mock_config.inference.default_model = (
+            "publishers/google/models/gemini-2.5-flash"
+        )
+        mock_config.inference.default_provider = "google-vertex"
+
+        mock_lsc = mocker.patch("app.endpoints.health.AsyncLlamaStackClientHolder")
+        mock_client = mocker.AsyncMock()
+        mock_lsc.return_value.get_client.return_value = mock_client
+        mock_client.models.list.side_effect = APIConnectionError(request=mocker.Mock())
+
+        available, reason = await check_default_model_available()
+
+        assert available is False
+        assert "Failed to check model availability" in reason
+
+    @pytest.mark.asyncio
+    async def test_api_status_error(self, mocker: MockerFixture) -> None:
+        """Test returns False when model list call fails with API status error."""
+        mock_config = mocker.patch("app.endpoints.health.configuration")
+        mock_config.inference.default_model = (
+            "publishers/google/models/gemini-2.5-flash"
+        )
+        mock_config.inference.default_provider = "google-vertex"
+
+        mock_lsc = mocker.patch("app.endpoints.health.AsyncLlamaStackClientHolder")
+        mock_client = mocker.AsyncMock()
+        mock_lsc.return_value.get_client.return_value = mock_client
+
+        mock_response = mocker.Mock()
+        mock_response.status_code = 500
+        mock_response.headers = {}
+        mock_client.models.list.side_effect = APIStatusError(
+            message="Internal error",
+            response=mock_response,
+            body=None,
+        )
+
+        available, reason = await check_default_model_available()
+
+        assert available is False
+        assert "API error checking model availability" in reason
+
+
+@pytest.mark.asyncio
+async def test_readiness_probe_fails_when_model_not_available(
+    mocker: MockerFixture,
+) -> None:
+    """Test readiness returns 503 when providers are healthy but default model is missing."""
+    mock_authorization_resolvers(mocker)
+
+    mock_get_providers = mocker.patch(
+        "app.endpoints.health.get_providers_health_statuses"
+    )
+    mock_get_providers.return_value = [
+        ProviderHealthStatus(
+            provider_id="provider1",
+            status=HealthStatus.OK.value,
+            message="Provider is healthy",
+        )
+    ]
+
+    mock_check_model = mocker.patch(
+        "app.endpoints.health.check_default_model_available"
+    )
+    mock_check_model.return_value = (
+        False,
+        "Default model google-vertex/publishers/google/models/gemini-2.5-flash "
+        "not found in model registry",
+    )
+
+    mock_response = mocker.Mock()
+    auth: AuthTuple = ("test_user_id", "test_user", True, "test_token")
+
+    response = await readiness_probe_get_method(auth=auth, response=mock_response)
+
+    assert response.ready is False
+    assert "not found in model registry" in response.reason
+    assert mock_response.status_code == 503
+
+
+@pytest.mark.asyncio
+async def test_readiness_probe_succeeds_with_healthy_providers_and_model(
+    mocker: MockerFixture,
+) -> None:
+    """Test readiness returns 200 when providers are healthy and default model is available."""
+    mock_authorization_resolvers(mocker)
+
+    mock_get_providers = mocker.patch(
+        "app.endpoints.health.get_providers_health_statuses"
+    )
+    mock_get_providers.return_value = [
+        ProviderHealthStatus(
+            provider_id="provider1",
+            status=HealthStatus.OK.value,
+            message="Provider is healthy",
+        )
+    ]
+
+    mock_check_model = mocker.patch(
+        "app.endpoints.health.check_default_model_available"
+    )
+    mock_check_model.return_value = (True, "Default model is available")
+
+    mock_response = mocker.Mock()
+    auth: AuthTuple = ("test_user_id", "test_user", True, "test_token")
+
+    response = await readiness_probe_get_method(auth=auth, response=mock_response)
+
+    assert response.ready is True
+    assert response.reason == "All providers are healthy"
