fix(rlsapi_v1): handle RuntimeError for context length in infer endpoint

- Add RuntimeError catch block matching query.py and streaming_query.py pattern
- Return 413 with PromptTooLongResponse when context_length error detected
- Re-raise non-context-length RuntimeErrors for proper middleware handling
- Add unit tests for both context_length and other RuntimeError scenarios

Signed-off-by: Major Hayden <major@redhat.com>

Author: major

src/app/endpoints/query.py

@@ -257,11 +257,13 @@ async def retrieve_response(  # pylint: disable=too-many-locals
         response = await client.responses.create(**responses_params.model_dump())
         response = cast(OpenAIResponseObject, response)
 
-    except RuntimeError as e:  # library mode wraps 413 into runtime error
+    except RuntimeError as e:  # library mode wraps HTTP errors as RuntimeError
         if "context_length" in str(e).lower():
             error_response = PromptTooLongResponse(model=responses_params.model)
             raise HTTPException(**error_response.model_dump()) from e
-        raise e
+        logger.exception("RuntimeError during inference")
+        error_response = InternalServerErrorResponse.generic()
+        raise HTTPException(**error_response.model_dump()) from e
     except APIConnectionError as e:
         error_response = ServiceUnavailableResponse(
             backend_name="Llama Stack",

src/app/endpoints/rlsapi_v1.py

@@ -24,6 +24,7 @@
 from models.responses import (
     ForbiddenResponse,
     InternalServerErrorResponse,
+    PromptTooLongResponse,
     QuotaExceededResponse,
     ServiceUnavailableResponse,
     UnauthorizedResponse,
@@ -270,6 +271,26 @@ async def infer_endpoint(
             input_source, instructions, tools=mcp_tools
         )
         inference_time = time.monotonic() - start_time
+    except RuntimeError as e:
+        # Library mode wraps HTTP errors as RuntimeError
+        inference_time = time.monotonic() - start_time
+        metrics.llm_calls_failures_total.inc()
+        _queue_splunk_event(
+            background_tasks,
+            infer_request,
+            request,
+            request_id,
+            str(e),
+            inference_time,
+            "infer_error",
+        )
+        if "context_length" in str(e).lower():
+            logger.error("Prompt too long for request %s: %s", request_id, e)
+            error_response = PromptTooLongResponse(model=_get_default_model_id())
+            raise HTTPException(**error_response.model_dump()) from e
+        logger.exception("RuntimeError during inference for request %s", request_id)
+        response = InternalServerErrorResponse.generic()
+        raise HTTPException(**response.model_dump()) from e
     except APIConnectionError as e:
         inference_time = time.monotonic() - start_time
         metrics.llm_calls_failures_total.inc()

src/app/endpoints/streaming_query.py

@@ -269,11 +269,13 @@ async def retrieve_response_generator(
         return response_generator(response, context, turn_summary), turn_summary
 
     # Handle know LLS client errors only at stream creation time and shield execution
-    except RuntimeError as e:  # library mode wraps 413 into runtime error
+    except RuntimeError as e:  # library mode wraps HTTP errors as RuntimeError
         if "context_length" in str(e).lower():
             error_response = PromptTooLongResponse(model=responses_params.model)
             raise HTTPException(**error_response.model_dump()) from e
-        raise e
+        logger.exception("RuntimeError during streaming inference")
+        error_response = InternalServerErrorResponse.generic()
+        raise HTTPException(**error_response.model_dump()) from e
     except APIConnectionError as e:
         error_response = ServiceUnavailableResponse(
             backend_name="Llama Stack",
@@ -407,9 +409,9 @@ async def response_generator(  # pylint: disable=too-many-branches,too-many-stat
     chunk_id = 0
     media_type = context.query_request.media_type or MEDIA_TYPE_JSON
     text_parts: list[str] = []
-    mcp_calls: dict[int, tuple[str, str]] = (
-        {}
-    )  # output_index -> (mcp_call_id, mcp_call_name)
+    mcp_calls: dict[
+        int, tuple[str, str]
+    ] = {}  # output_index -> (mcp_call_id, mcp_call_name)
     latest_response_object: Optional[OpenAIResponseObject] = None
 
     logger.debug("Starting streaming response (Responses API) processing")

tests/unit/app/endpoints/test_query.py

@@ -7,7 +7,7 @@
 from typing import Any
 
 import pytest
-from fastapi import HTTPException, Request
+from fastapi import HTTPException, Request, status
 from llama_stack_api.openai_responses import OpenAIResponseObject
 from llama_stack_client import APIConnectionError, APIStatusError, AsyncLlamaStackClient
 from pytest_mock import MockerFixture
@@ -110,9 +110,7 @@ async def test_successful_query_no_conversation(
         mocker: MockerFixture,
     ) -> None:
         """Test successful query without existing conversation."""
-        query_request = QueryRequest(
-            query="What is Kubernetes?"
-        )  # pyright: ignore[reportCallIssue]
+        query_request = QueryRequest(query="What is Kubernetes?")  # pyright: ignore[reportCallIssue]
 
         mocker.patch("app.endpoints.query.configuration", setup_configuration)
         mocker.patch("app.endpoints.query.check_configuration_loaded")
@@ -386,9 +384,7 @@ async def test_query_azure_token_refresh(
         mocker: MockerFixture,
     ) -> None:
         """Test query refreshes Azure token when needed."""
-        query_request = QueryRequest(
-            query="What is Kubernetes?"
-        )  # pyright: ignore[reportCallIssue]
+        query_request = QueryRequest(query="What is Kubernetes?")  # pyright: ignore[reportCallIssue]
 
         mocker.patch("app.endpoints.query.configuration", setup_configuration)
         mocker.patch("app.endpoints.query.check_configuration_loaded")
@@ -659,8 +655,9 @@ async def test_retrieve_response_runtime_error_other(
             side_effect=RuntimeError("Some other error")
         )
 
-        with pytest.raises(RuntimeError):
+        with pytest.raises(HTTPException) as exc_info:
             await retrieve_response(mock_client, mock_responses_params)
+        assert exc_info.value.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR
 
     @pytest.mark.asyncio
     async def test_retrieve_response_with_tool_calls(

tests/unit/app/endpoints/test_rlsapi_v1.py

@@ -126,6 +126,26 @@ def mock_api_connection_error_fixture(mocker: MockerFixture) -> None:
     )
 
 
+@pytest.fixture(name="mock_runtime_error_context_length")
+def mock_runtime_error_context_length_fixture(mocker: MockerFixture) -> None:
+    """Mock responses.create() to raise RuntimeError with context_length message."""
+    _setup_responses_mock(
+        mocker,
+        mocker.AsyncMock(
+            side_effect=RuntimeError("context_length exceeded maximum tokens")
+        ),
+    )
+
+
+@pytest.fixture(name="mock_runtime_error_other")
+def mock_runtime_error_other_fixture(mocker: MockerFixture) -> None:
+    """Mock responses.create() to raise RuntimeError with non-context_length message."""
+    _setup_responses_mock(
+        mocker,
+        mocker.AsyncMock(side_effect=RuntimeError("Some other runtime error")),
+    )
+
+
 # --- Test _build_instructions ---
 
 
@@ -400,6 +420,51 @@ async def test_infer_api_connection_error_returns_503(
     assert exc_info.value.status_code == status.HTTP_503_SERVICE_UNAVAILABLE
 
 
+@pytest.mark.asyncio
+async def test_infer_runtime_error_context_length_returns_413(
+    mocker: MockerFixture,
+    mock_configuration: AppConfig,
+    mock_runtime_error_context_length: None,
+    mock_auth_resolvers: None,
+) -> None:
+    """Test /infer returns 413 when LLM raises RuntimeError with context_length."""
+    infer_request = RlsapiV1InferRequest(question="Test question")
+    mock_request = _create_mock_request(mocker)
+    mock_background_tasks = _create_mock_background_tasks(mocker)
+
+    with pytest.raises(HTTPException) as exc_info:
+        await infer_endpoint(
+            infer_request=infer_request,
+            request=mock_request,
+            background_tasks=mock_background_tasks,
+            auth=MOCK_AUTH,
+        )
+
+    assert exc_info.value.status_code == status.HTTP_413_REQUEST_ENTITY_TOO_LARGE
+
+
+@pytest.mark.asyncio
+async def test_infer_runtime_error_other_reraises(
+    mocker: MockerFixture,
+    mock_configuration: AppConfig,
+    mock_runtime_error_other: None,
+    mock_auth_resolvers: None,
+) -> None:
+    """Test /infer returns 500 for RuntimeError when not context_length related."""
+    infer_request = RlsapiV1InferRequest(question="Test question")
+    mock_request = _create_mock_request(mocker)
+    mock_background_tasks = _create_mock_background_tasks(mocker)
+
+    with pytest.raises(HTTPException) as exc_info:
+        await infer_endpoint(
+            infer_request=infer_request,
+            request=mock_request,
+            background_tasks=mock_background_tasks,
+            auth=MOCK_AUTH,
+        )
+    assert exc_info.value.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR
+
+
 @pytest.mark.asyncio
 async def test_infer_empty_llm_response_returns_fallback(
     mocker: MockerFixture,

tests/unit/app/endpoints/test_streaming_query.py

@@ -6,7 +6,7 @@
 from typing import Any
 
 import pytest
-from fastapi import HTTPException, Request
+from fastapi import HTTPException, Request, status
 from fastapi.responses import StreamingResponse
 from llama_stack_api.openai_responses import (
     OpenAIResponseObject,
@@ -257,20 +257,14 @@ class TestOLSCompatibilityIntegration:
 
     def test_media_type_validation(self) -> None:
         """Test that media type validation works correctly."""
-        valid_request = QueryRequest(
-            query="test", media_type="application/json"
-        )  # pyright: ignore[reportCallIssue]
+        valid_request = QueryRequest(query="test", media_type="application/json")  # pyright: ignore[reportCallIssue]
         assert valid_request.media_type == "application/json"
 
-        valid_request = QueryRequest(
-            query="test", media_type="text/plain"
-        )  # pyright: ignore[reportCallIssue]
+        valid_request = QueryRequest(query="test", media_type="text/plain")  # pyright: ignore[reportCallIssue]
         assert valid_request.media_type == "text/plain"
 
         with pytest.raises(ValueError, match="media_type must be either"):
-            QueryRequest(
-                query="test", media_type="invalid/type"
-            )  # pyright: ignore[reportCallIssue]
+            QueryRequest(query="test", media_type="invalid/type")  # pyright: ignore[reportCallIssue]
 
     def test_ols_end_event_structure(self) -> None:
         """Test that end event follows OLS structure."""
@@ -322,9 +316,7 @@ async def test_successful_streaming_query(
         mocker: MockerFixture,
     ) -> None:
         """Test successful streaming query."""
-        query_request = QueryRequest(
-            query="What is Kubernetes?"
-        )  # pyright: ignore[reportCallIssue]
+        query_request = QueryRequest(query="What is Kubernetes?")  # pyright: ignore[reportCallIssue]
 
         mocker.patch("app.endpoints.streaming_query.configuration", setup_configuration)
         mocker.patch("app.endpoints.streaming_query.check_configuration_loaded")
@@ -574,9 +566,7 @@ async def test_streaming_query_azure_token_refresh(
         mocker: MockerFixture,
     ) -> None:
         """Test streaming query refreshes Azure token when needed."""
-        query_request = QueryRequest(
-            query="What is Kubernetes?"
-        )  # pyright: ignore[reportCallIssue]
+        query_request = QueryRequest(query="What is Kubernetes?")  # pyright: ignore[reportCallIssue]
 
         mocker.patch("app.endpoints.streaming_query.configuration", setup_configuration)
         mocker.patch("app.endpoints.streaming_query.check_configuration_loaded")
@@ -679,9 +669,7 @@ async def test_retrieve_response_generator_success(
 
         mock_context = mocker.Mock(spec=ResponseGeneratorContext)
         mock_context.client = mock_client
-        mock_context.query_request = QueryRequest(
-            query="test"
-        )  # pyright: ignore[reportCallIssue]
+        mock_context.query_request = QueryRequest(query="test")  # pyright: ignore[reportCallIssue]
 
         async def mock_response_gen() -> AsyncIterator[str]:
             yield "test"
@@ -769,9 +757,7 @@ async def test_retrieve_response_generator_connection_error(
 
         mock_context = mocker.Mock(spec=ResponseGeneratorContext)
         mock_context.client = mock_client
-        mock_context.query_request = QueryRequest(
-            query="test"
-        )  # pyright: ignore[reportCallIssue]
+        mock_context.query_request = QueryRequest(query="test")  # pyright: ignore[reportCallIssue]
 
         mocker.patch(
             "app.endpoints.streaming_query.run_shield_moderation",
@@ -822,9 +808,7 @@ async def test_retrieve_response_generator_api_status_error(
 
         mock_context = mocker.Mock(spec=ResponseGeneratorContext)
         mock_context.client = mock_client
-        mock_context.query_request = QueryRequest(
-            query="test"
-        )  # pyright: ignore[reportCallIssue]
+        mock_context.query_request = QueryRequest(query="test")  # pyright: ignore[reportCallIssue]
 
         mocker.patch(
             "app.endpoints.streaming_query.run_shield_moderation",
@@ -872,9 +856,7 @@ async def test_retrieve_response_generator_runtime_error_context_length(
 
         mock_context = mocker.Mock(spec=ResponseGeneratorContext)
         mock_context.client = mock_client
-        mock_context.query_request = QueryRequest(
-            query="test"
-        )  # pyright: ignore[reportCallIssue]
+        mock_context.query_request = QueryRequest(query="test")  # pyright: ignore[reportCallIssue]
 
         mocker.patch(
             "app.endpoints.streaming_query.run_shield_moderation",
@@ -919,9 +901,7 @@ async def test_retrieve_response_generator_runtime_error_other(
 
         mock_context = mocker.Mock(spec=ResponseGeneratorContext)
         mock_context.client = mock_client
-        mock_context.query_request = QueryRequest(
-            query="test"
-        )  # pyright: ignore[reportCallIssue]
+        mock_context.query_request = QueryRequest(query="test")  # pyright: ignore[reportCallIssue]
 
         mocker.patch(
             "app.endpoints.streaming_query.run_shield_moderation",
@@ -932,8 +912,9 @@ async def test_retrieve_response_generator_runtime_error_other(
             side_effect=RuntimeError("Some other error")
         )
 
-        with pytest.raises(RuntimeError):
+        with pytest.raises(HTTPException) as exc_info:
             await retrieve_response_generator(mock_responses_params, mock_context)
+        assert exc_info.value.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR
 
 
 class TestGenerateResponse:
@@ -950,9 +931,7 @@ async def mock_generator() -> AsyncIterator[str]:
         mock_context = mocker.Mock(spec=ResponseGeneratorContext)
         mock_context.conversation_id = "conv_123"
         mock_context.user_id = "user_123"
-        mock_context.query_request = QueryRequest(
-            query="test"
-        )  # pyright: ignore[reportCallIssue]
+        mock_context.query_request = QueryRequest(query="test")  # pyright: ignore[reportCallIssue]
         mock_context.started_at = "2024-01-01T00:00:00Z"
         mock_context.skip_userid_check = False
 
@@ -1047,9 +1026,7 @@ async def mock_generator() -> AsyncIterator[str]:
 
         mock_context = mocker.Mock(spec=ResponseGeneratorContext)
         mock_context.conversation_id = "conv_123"
-        mock_context.query_request = QueryRequest(
-            query="test"
-        )  # pyright: ignore[reportCallIssue]
+        mock_context.query_request = QueryRequest(query="test")  # pyright: ignore[reportCallIssue]
         mock_context.started_at = "2024-01-01T00:00:00Z"
         mock_context.skip_userid_check = False
 
@@ -1082,9 +1059,7 @@ async def mock_generator() -> AsyncIterator[str]:
 
         mock_context = mocker.Mock(spec=ResponseGeneratorContext)
         mock_context.conversation_id = "conv_123"
-        mock_context.query_request = QueryRequest(
-            query="test"
-        )  # pyright: ignore[reportCallIssue]
+        mock_context.query_request = QueryRequest(query="test")  # pyright: ignore[reportCallIssue]
         mock_context.started_at = "2024-01-01T00:00:00Z"
         mock_context.skip_userid_check = False