Improve K8s authentication error handling

Refactored Kubernetes authentication error handling to replace the
generic `ClusterIDUnavailableError` with a granular exception hierarchy
that differentiates between transient API failures (503) and persistent
configuration issues (500). Error messages are now preserved and returned
to clients with appropriate HTTP status codes.

**Motivation**
The current implementation had several issues:
- Single `ClusterIDUnavailableError` for all failure scenarios
- Error context lost when converting to generic 500 responses
- Always returned HTTP 500, even for transient K8s API failures (should be 503)
- Broad `except Exception` catches masked specific errors
- Hard to debug production issues due to generic error messages

Note: This PR does not introduce any breaking changes. The changes are internal to the
K8s authentication module and the public API __call__ method signature remains unchanged.

Signed-off-by: Anik Bhattacharjee <anbhatta@redhat.com>

Author: anik120

src/authentication/k8s.py

@@ -2,6 +2,7 @@
 
 import os
 from pathlib import Path
+from http import HTTPStatus
 from typing import Optional, Self
 
 import kubernetes.client
@@ -30,8 +31,45 @@
 )
 
 
-class ClusterIDUnavailableError(Exception):
-    """Cluster ID is not available."""
+class K8sAuthenticationError(Exception):
+    """Base exception for Kubernetes authentication errors."""
+
+
+class K8sAPIConnectionError(K8sAuthenticationError):
+    """Cannot connect to Kubernetes API server.
+
+    Indicates transient failures that may be resolved by retrying.
+    Maps to HTTP 503 Service Unavailable.
+    """
+
+
+class K8sConfigurationError(K8sAuthenticationError):
+    """Kubernetes cluster configuration issue.
+
+    Indicates persistent configuration problems requiring admin intervention.
+    Maps to HTTP 500 Internal Server Error.
+    """
+
+
+class ClusterVersionNotFoundError(K8sConfigurationError):
+    """ClusterVersion resource not found in OpenShift cluster.
+
+    Raised when the ClusterVersion custom resource does not exist (HTTP 404).
+    """
+
+
+class ClusterVersionPermissionError(K8sConfigurationError):
+    """No permission to access ClusterVersion resource.
+
+    Raised when RBAC denies access to the ClusterVersion resource (HTTP 403).
+    """
+
+
+class InvalidClusterVersionError(K8sConfigurationError):
+    """ClusterVersion resource has invalid structure or missing required fields.
+
+    Raised when the ClusterVersion exists but is missing spec.clusterID or has wrong type.
+    """
 
 
 class K8sClientSingleton:
@@ -156,9 +194,12 @@ def _get_cluster_id(cls) -> str:
             str: The cluster's `clusterID`.
 
         Raises:
-            ClusterIDUnavailableError: If the cluster ID cannot be obtained due
-            to missing keys, an API error, or any unexpected error.
+            K8sAPIConnectionError: If the Kubernetes API is unreachable or returns 5xx errors.
+            ClusterVersionNotFoundError: If the ClusterVersion resource does not exist (404).
+            ClusterVersionPermissionError: If access to ClusterVersion is denied (403).
+            InvalidClusterVersionError: If ClusterVersion has invalid structure or missing fields.
         """
+        version_data: Optional[object] = None
         try:
             custom_objects_api = cls.get_custom_objects_api()
             version_data = custom_objects_api.get_cluster_custom_object(
@@ -167,22 +208,59 @@ def _get_cluster_id(cls) -> str:
             cluster_id = version_data["spec"]["clusterID"]
             cls._cluster_id = cluster_id
             return cluster_id
+        except ApiException as e:
+            # Handle specific HTTP status codes from Kubernetes API
+            if e.status == HTTPStatus.NOT_FOUND:
+                logger.error(
+                    "ClusterVersion resource 'version' not found in cluster: %s",
+                    e.reason,
+                )
+                raise ClusterVersionNotFoundError(
+                    "ClusterVersion 'version' resource not found in OpenShift cluster"
+                ) from e
+            if e.status == HTTPStatus.FORBIDDEN:
+                logger.error(
+                    "Permission denied to access ClusterVersion resource: %s", e.reason
+                )
+                raise ClusterVersionPermissionError(
+                    "Insufficient permissions to read ClusterVersion resource"
+                ) from e
+            # Classify errors by status code range
+            # 5xx errors and 429 (rate limit) are transient - map to 503
+            if e.status >= 500 or e.status == HTTPStatus.TOO_MANY_REQUESTS:
+                logger.error(
+                    "Kubernetes API unavailable while fetching ClusterVersion (status %s): %s",
+                    e.status,
+                    e.reason,
+                )
+                raise K8sAPIConnectionError(
+                    f"Failed to connect to Kubernetes API: {e.reason} (status {e.status})"
+                ) from e
+            # All other errors (4xx client errors) are configuration issues - map to 500
+            logger.error(
+                "Kubernetes API returned client error while fetching "
+                "ClusterVersion (status %s): %s",
+                e.status,
+                e.reason,
+            )
+            raise K8sConfigurationError(
+                f"Kubernetes API request failed: {e.reason} (status {e.status})"
+            ) from e
         except KeyError as e:
             logger.error(
-                "Failed to get cluster_id from cluster, missing keys in version object"
+                "ClusterVersion missing required field 'spec.clusterID': %s", e
             )
-            raise ClusterIDUnavailableError("Failed to get cluster ID") from e
+            raise InvalidClusterVersionError(
+                f"ClusterVersion missing required field: {e}"
+            ) from e
         except TypeError as e:
             logger.error(
-                "Failed to get cluster_id, version object is: %s", version_data
+                "ClusterVersion has invalid structure, version_data type: %s",
+                type(version_data).__name__ if version_data is not None else "unknown",
             )
-            raise ClusterIDUnavailableError("Failed to get cluster ID") from e
-        except ApiException as e:
-            logger.error("API exception during ClusterInfo: %s", e)
-            raise ClusterIDUnavailableError("Failed to get cluster ID") from e
-        except Exception as e:
-            logger.error("Unexpected error during getting cluster ID: %s", e)
-            raise ClusterIDUnavailableError("Failed to get cluster ID") from e
+            raise InvalidClusterVersionError(
+                f"ClusterVersion has invalid type or structure: {e}"
+            ) from e
 
     @classmethod
     def get_cluster_id(cls) -> str:
@@ -199,7 +277,10 @@ def get_cluster_id(cls) -> str:
             str: The cluster identifier.
 
         Raises:
-            ClusterIDUnavailableError: If running in-cluster and fetching the cluster ID fails.
+            K8sAPIConnectionError: If the Kubernetes API is unreachable.
+            ClusterVersionNotFoundError: If the ClusterVersion resource does not exist.
+            ClusterVersionPermissionError: If access to ClusterVersion is denied.
+            InvalidClusterVersionError: If ClusterVersion has invalid structure.
         """
         if cls._instance is None:
             cls()
@@ -310,11 +391,20 @@ async def __call__(self, request: Request) -> tuple[str, str, bool, str]:
         if user_info.user.username == "kube:admin":
             try:
                 user_info.user.uid = K8sClientSingleton.get_cluster_id()
-            except ClusterIDUnavailableError as e:
-                logger.error("Failed to get cluster ID: %s", e)
+            except K8sAPIConnectionError as e:
+                # Kubernetes API is unreachable - return 503
+                logger.error("Cannot connect to Kubernetes API: %s", e)
+                response = ServiceUnavailableResponse(
+                    backend_name="Kubernetes API",
+                    cause=str(e),
+                )
+                raise HTTPException(**response.model_dump()) from e
+            except K8sConfigurationError as e:
+                # Cluster misconfiguration or client error - return 500
+                logger.error("Cluster configuration error: %s", e)
                 response = InternalServerErrorResponse(
                     response="Internal server error",
-                    cause="Unable to retrieve cluster ID",
+                    cause=str(e),
                 )
                 raise HTTPException(**response.model_dump()) from e
 

src/models/responses.py

@@ -2414,6 +2414,27 @@ class InternalServerErrorResponse(AbstractErrorResponse):
                         "cause": "Failed to query the database",
                     },
                 },
+                {
+                    "label": "cluster version not found",
+                    "detail": {
+                        "response": "Internal server error",
+                        "cause": "ClusterVersion 'version' resource not found in OpenShift cluster",
+                    },
+                },
+                {
+                    "label": "cluster version permission denied",
+                    "detail": {
+                        "response": "Internal server error",
+                        "cause": "Insufficient permissions to read ClusterVersion resource",
+                    },
+                },
+                {
+                    "label": "invalid cluster version",
+                    "detail": {
+                        "response": "Internal server error",
+                        "cause": "ClusterVersion missing required field: 'clusterID'",
+                    },
+                },
             ]
         }
     }
@@ -2537,7 +2558,17 @@ class ServiceUnavailableResponse(AbstractErrorResponse):
                         "response": "Unable to connect to Llama Stack",
                         "cause": "Connection error while trying to reach backend service.",
                     },
-                }
+                },
+                {
+                    "label": "kubernetes api",
+                    "detail": {
+                        "response": "Unable to connect to Kubernetes API",
+                        "cause": (
+                            "Failed to connect to Kubernetes API: "
+                            "Service Unavailable (status 503)"
+                        ),
+                    },
+                },
             ]
         }
     }