LCORE-1859: Enhance /readiness endpoint with degraded mode reporting

anik120 · anik120 · commit 9bd1ce3acd46 · 2026-06-18T10:53:51.000-04:00
#1781 introduced "degraded mode support" - the ability to start lightspeed-stack and keep it running even when llama-stack server might not be available. This PR adds comprehensive degraded mode status reporting to the /readiness endpoint while maintaining clean API boundaries and Kubernetes probe semantics. - Enhanced HealthStatus enum with DEGRADED and UNHEALTHY service-level statuses while preserving provider-level statuses (OK, ERROR, NOT_IMPLEMENTED, UNKNOWN) - Enhanced /readiness endpoint to return 200 (ready=true) in degraded mode following Kubernetes semantics; only returns 503 when truly unhealthy - Refactored to avoid leaking implementation details in API responses: * Removed llama_stack field from ReadinessResponse * Removed Llama Stack version tracking from DegradedModeTracker * Focus on functional impacts rather than internal technology stack This design keeps internal implementation details (Llama Stack) private while exposing clear functional impacts to API consumers. Signed-off-by: Anik Bhattacharjee <anbhatta@redhat.com>
diff --git a/src/app/endpoints/health.py b/src/app/endpoints/health.py
@@ -26,8 +26,12 @@
     LivenessResponse,
     ReadinessResponse,
 )
-from models.common import HealthStatus, ProviderHealthStatus
+from models.common import (
+    HealthStatus,
+    ProviderHealthStatus,
+)
 from models.config import Action
+from utils.degraded_mode import DegradedModeTracker
 
 logger = get_logger(__name__)
 router = APIRouter(tags=["health"])
@@ -117,11 +121,11 @@ async def readiness_probe_get_method(
     response: Response,
 ) -> ReadinessResponse:
     """
-    Handle the readiness probe endpoint, returning service readiness.
+    Handle the readiness probe endpoint, returning service readiness and health status.
 
-    If any provider reports an error status, responds with HTTP 503
-    and details of unhealthy providers; otherwise, indicates the
-    service is ready.
+    Returns comprehensive health information including overall service status,
+    provider health, and functional impacts. The service is considered "ready" even
+    in degraded mode (returns 200), but reports reduced functionality.
 
     ### Parameters:
     - response: The outgoing HTTP response (used by middleware).
@@ -130,47 +134,82 @@ async def readiness_probe_get_method(
     ### Raises:
     - HTTPException: with status 401 for unauthorized access.
     - HTTPException: with status 403 if permission is denied.
-    - HTTPException: with status 500 and a detail object containing `response`
-      and `cause` when service configuration is wrong or incomplete.
-    - HTTPException: with status 503 and a detail object containing `response`
-      and `cause` when unable to connect to Llama Stack.
+    - HTTPException: with status 503 when service is unhealthy (providers down,
+      models unavailable) and degraded mode is not enabled.
 
     ### Returns:
-    - ReadinessResponse: Object with `ready` indicating overall readiness,
-      `reason` explaining the outcome, and `providers` containing the list of
-      unhealthy ProviderHealthStatus entries (empty when ready).
+    - ReadinessResponse: Object with comprehensive health status including:
+      - ready: True if service can handle requests (even in degraded mode)
+      - reason: Description of service state
+      - overall_status: healthy, degraded, or unhealthy
+      - impacts: Functional limitations when degraded/unhealthy
+      - providers: List of unhealthy providers
     """
     # Used only for authorization
     _ = auth
 
-    logger.info("Response to /v1/readiness endpoint")
+    logger.info("Response to /readiness endpoint")
 
-    provider_statuses = await get_providers_health_statuses()
+    degraded_tracker = DegradedModeTracker()
+    is_degraded = degraded_tracker.is_degraded()
 
-    # Check if any provider is unhealthy (not counting not_implemented as unhealthy)
+    # Determine overall status
+    if is_degraded:
+        # Service is ready (can serve health checks, metrics, etc.) but degraded
+        impacts = [
+            "LLM inference unavailable",
+            "RAG functionality unavailable",
+            "Agent tools unavailable",
+        ]
+        return ReadinessResponse(
+            ready=True,
+            reason="Service running in degraded mode",
+            overall_status=HealthStatus.DEGRADED,
+            impacts=impacts,
+            providers=[],
+        )
+
+    # Not in degraded mode - check provider health
+    provider_statuses = await get_providers_health_statuses()
     unhealthy_providers = [
         p for p in provider_statuses if p.status == HealthStatus.ERROR.value
     ]
 
     if unhealthy_providers:
-        ready = False
         unhealthy_provider_names = [p.provider_id for p in unhealthy_providers]
         reason = f"Providers not healthy: {', '.join(unhealthy_provider_names)}"
+        impacts = [
+            f"Provider {p.provider_id} unhealthy: {p.message}"
+            for p in unhealthy_providers
+        ]
         response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
         return ReadinessResponse(
-            ready=ready, reason=reason, providers=unhealthy_providers
+            ready=False,
+            reason=reason,
+            overall_status=HealthStatus.UNHEALTHY,
+            impacts=impacts,
+            providers=unhealthy_providers,
         )
 
     # Check that the default model is registered in the model registry
     model_available, model_reason = await check_default_model_available()
     if not model_available:
         response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
         return ReadinessResponse(
-            ready=False, reason=model_reason, providers=unhealthy_providers
+            ready=False,
+            reason=model_reason,
+            overall_status=HealthStatus.UNHEALTHY,
+            impacts=["Default model not available in registry"],
+            providers=[],
         )
 
+    # All healthy
     return ReadinessResponse(
-        ready=True, reason="All providers are healthy", providers=unhealthy_providers
+        ready=True,
+        reason="All providers are healthy",
+        overall_status=HealthStatus.HEALTHY,
+        impacts=None,
+        providers=[],
     )
 
 
diff --git a/src/app/main.py b/src/app/main.py
@@ -26,6 +26,7 @@
 from models.api.responses.error import InternalServerErrorResponse
 from sentry import initialize_sentry
 from utils.common import register_mcp_servers_async
+from utils.degraded_mode import DegradedModeTracker
 from utils.llama_stack_version import check_llama_stack_version
 
 logger = get_logger(__name__)
@@ -81,15 +82,19 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
     await AsyncLlamaStackClientHolder().load(llama_stack_config)
     client: AsyncLlamaStackClient = AsyncLlamaStackClientHolder().get_client()
     logger.debug("Llama Stack client initialized, trying to connect to Llama Stack")
-    # check if the Llama Stack version is supported by the service
+    # Check connectivity to Llama Stack and set degraded mode if unavailable
+    degraded_tracker = DegradedModeTracker()
     try:
         llama_stack_version = await check_llama_stack_version(
             client, llama_stack_config.max_retries, llama_stack_config.retry_delay
         )
         if llama_stack_version is None:
             logger.error("Cannot retrieve Llama Stack version, check connection")
+            if llama_stack_config.allow_degraded_mode:
+                degraded_tracker.set_degraded("Llama Stack connection check failed")
         else:
             logger.debug("Llama Stack version: %s", llama_stack_version)
+            degraded_tracker.set_healthy()
     except APIConnectionError as e:
         # if degraded mode is allowed, simply ignore the exception
         llama_stack_url = llama_stack_config.url
@@ -103,6 +108,7 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
         )
         if llama_stack_config.allow_degraded_mode:
             logger.info("Entering degraded mode: LCORE running w/o Llama Stack")
+            degraded_tracker.set_degraded(f"Failed to connect to Llama Stack: {e!s}")
         else:
             raise
 
diff --git a/src/models/api/responses/successful/probes.py b/src/models/api/responses/successful/probes.py
@@ -1,11 +1,14 @@
 """Successful probe-related API responses (info, readiness, liveness, status, auth)."""
 
-from typing import Any
+from typing import Any, Optional
 
 from pydantic import Field
 
 from models.api.responses.successful.bases import AbstractSuccessfulResponse
-from models.common.health import ProviderHealthStatus
+from models.common.health import (
+    HealthStatus,
+    ProviderHealthStatus,
+)
 
 
 class InfoResponse(AbstractSuccessfulResponse):
@@ -50,26 +53,46 @@ class ReadinessResponse(AbstractSuccessfulResponse):
     """Model representing response to a readiness request.
 
     Attributes:
-        ready: If service is ready.
-        reason: The reason for the readiness.
-        providers: List of unhealthy providers in case of readiness failure.
+        ready: If service is ready to handle requests.
+        reason: The reason for the readiness status.
+        overall_status: Overall service health status (healthy/degraded/unhealthy).
+        impacts: Optional list of functional impacts when degraded or unhealthy.
+        providers: List of unhealthy providers (empty when all healthy).
     """
 
     ready: bool = Field(
         ...,
-        description="Flag indicating if service is ready",
+        description="Flag indicating if service is ready to handle requests",
         examples=[True, False],
     )
 
     reason: str = Field(
         ...,
-        description="The reason for the readiness",
+        description="The reason for the readiness status",
         examples=["Service is ready"],
     )
 
+    overall_status: HealthStatus = Field(
+        ...,
+        description="Overall service health status",
+        examples=["healthy", "degraded", "unhealthy"],
+    )
+
+    impacts: Optional[list[str]] = Field(
+        None,
+        description="List of functional impacts when service is degraded or unhealthy",
+        examples=[
+            [
+                "LLM inference unavailable",
+                "RAG functionality unavailable",
+                "Agent tools unavailable",
+            ]
+        ],
+    )
+
     providers: list[ProviderHealthStatus] = Field(
         ...,
-        description="List of unhealthy providers in case of readiness failure.",
+        description="List of unhealthy providers (empty when all healthy)",
         examples=[],
     )
 
@@ -79,9 +102,22 @@ class ReadinessResponse(AbstractSuccessfulResponse):
             "examples": [
                 {
                     "ready": True,
-                    "reason": "Service is ready",
+                    "reason": "All providers are healthy",
+                    "overall_status": "healthy",
+                    "impacts": None,
                     "providers": [],
-                }
+                },
+                {
+                    "ready": True,
+                    "reason": "Service running in degraded mode",
+                    "overall_status": "degraded",
+                    "impacts": [
+                        "LLM inference unavailable",
+                        "RAG functionality unavailable",
+                        "Agent tools unavailable",
+                    ],
+                    "providers": [],
+                },
             ]
         }
     }
diff --git a/src/models/common/__init__.py b/src/models/common/__init__.py
@@ -7,7 +7,10 @@
     Message,
 )
 from models.common.feedback import FeedbackCategory
-from models.common.health import HealthStatus, ProviderHealthStatus
+from models.common.health import (
+    HealthStatus,
+    ProviderHealthStatus,
+)
 from models.common.mcp import MCPServerAuthInfo, MCPServerInfo
 from models.common.moderation import (
     ShieldModerationBlocked,
diff --git a/src/models/common/health.py b/src/models/common/health.py
@@ -7,14 +7,33 @@
 
 
 class HealthStatus(str, Enum):
-    """Health status enum for provider health checks."""
+    """Health status enum for provider and service health checks.
 
+    This enum serves two purposes:
+
+    1. Provider-level health (returned by Llama Stack providers):
+       - OK: Provider is healthy and operational
+       - ERROR: Provider is unhealthy or failed health check
+       - NOT_IMPLEMENTED: Provider does not implement health checks
+       - UNKNOWN: Fallback when provider status cannot be determined
+
+    2. Service-level health (overall LCORE status):
+       - HEALTHY: All systems operational, LLS connected, all providers healthy
+       - DEGRADED: Service running with reduced functionality (e.g., LLS unavailable)
+       - UNHEALTHY: Service connected but one or more providers are unhealthy
+    """
+
+    # Provider-level statuses (from Llama Stack)
     OK = "ok"
     ERROR = "Error"
     NOT_IMPLEMENTED = "not_implemented"
-    HEALTHY = "healthy"
     UNKNOWN = "unknown"
 
+    # Service-level statuses (LCORE overall health)
+    HEALTHY = "healthy"
+    DEGRADED = "degraded"
+    UNHEALTHY = "unhealthy"
+
 
 class ProviderHealthStatus(BaseModel):
     """Model representing the health status of a provider.
@@ -35,5 +54,5 @@ class ProviderHealthStatus(BaseModel):
     message: Optional[str] = Field(
         None,
         description="Optional message about the health status",
-        examples=["All systems operational", "Llama Stack is unavailable"],
+        examples=["All systems operational", "Provider is unavailable"],
     )
diff --git a/src/utils/degraded_mode.py b/src/utils/degraded_mode.py
@@ -0,0 +1,53 @@
+"""Degraded mode state tracking.
+
+This module provides a singleton to track whether Lightspeed Core Stack is
+running in degraded mode (i.e., without Llama Stack connectivity).
+"""
+
+from typing import Optional
+
+from utils.types import Singleton
+
+
+class DegradedModeTracker(metaclass=Singleton):
+    """Track degraded mode state for Lightspeed Core Stack.
+
+    When LCORE cannot connect to Llama Stack during startup and
+    allow_degraded_mode is enabled, the service enters degraded mode.
+    This tracker maintains that state for health reporting.
+    """
+
+    def __init__(self) -> None:
+        """Initialize the degraded mode tracker."""
+        self._is_degraded: bool = False
+        self._degraded_reason: Optional[str] = None
+
+    def set_degraded(self, reason: str) -> None:
+        """Mark the service as running in degraded mode.
+
+        Parameters:
+            reason: Description of why degraded mode was entered.
+        """
+        self._is_degraded = True
+        self._degraded_reason = reason
+
+    def set_healthy(self) -> None:
+        """Mark the service as running in healthy mode."""
+        self._is_degraded = False
+        self._degraded_reason = None
+
+    def is_degraded(self) -> bool:
+        """Check if the service is running in degraded mode.
+
+        Returns:
+            True if service is in degraded mode, False otherwise.
+        """
+        return self._is_degraded
+
+    def get_degraded_reason(self) -> Optional[str]:
+        """Get the reason for degraded mode.
+
+        Returns:
+            Description of why degraded mode was entered, or None if healthy.
+        """
+        return self._degraded_reason
diff --git a/tests/unit/app/endpoints/test_health.py b/tests/unit/app/endpoints/test_health.py
diff --git a/tests/unit/test_degraded_mode.py b/tests/unit/test_degraded_mode.py
