feat: add metrics CLI filter, skip field, retry improvements, and infer endpoint (#222)

* Add --metrics CLI flag to filter which metrics run

Allows running a subset of configured metrics without editing YAML configs.
Example: --metrics custom:answer_correctness to skip RAGAS metrics.

* Add skip field to disable conversations with a reason

Adds skip and skip_reason fields to EvaluationData. Conversations
with skip: true are silently excluded during loading. skip_reason
is documentation-only — it stays in the YAML for humans to read.

* feat: add retry for all server errors and /infer endpoint support

Broaden retry logic from HTTP 429 only to include 5xx server errors,
enabling automatic retry with exponential backoff for transient
server failures.

Add RLSAPI /v1/infer endpoint support for tool call and RAG chunk
metadata retrieval, used by RHEL Lightspeed backend testing.

* config: increase default retry attempts from 3 to 5

Provides more resilience against transient server failures,
especially during long evaluation runs.

* style: apply black formatting to client and tests

* fix: address PR review feedback (asamal4 + CodeRabbit)

- Revert DEFAULT_LLM_RETRIES from 5 to 3
- Narrow retry codes to (429, 502, 503, 504), exclude 500
- Use RLSAPI native fields (name/args) in _rlsapi_infer_query
- Fix RAG chunk accumulation across multiple mcp_call results
- Redact prompt from debug log, log only metadata
- Add comment about extra_request_params not forwarded to /infer
- Fix tool result capture: use content with status fallback
- Update endpoint_type description to include infer
- Move skip tests from TestFilterByScope to TestDataValidator
- Fix MockerFixture import in test_validator.py
- Fix --metrics filter: handle turn_metrics=None by materializing
  system defaults before filtering; add conversation_metrics filter
- Add metrics=None to runner test fixture for --metrics support
- Add tests for metrics filter materialization

Signed-off-by: Ellis Low <elow@redhat.com>

* refactor: reduce complexity in client.py and validator.py to fix pylint

* fix: validate tool_results content before splitting in infer response

* fix: split test_client.py to satisfy pylint line limit

* fix: address CodeRabbit review comments

- Mock time.sleep in 5 retry tests to prevent real exponential backoff delays
- Update test_is_retryable_server_error docstring to mention transient 5xx errors

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>

---------

Signed-off-by: Ellis Low <elow@redhat.com>
Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>

Author: Lifto

src/lightspeed_evaluation/core/api/client.py

@@ -27,12 +27,23 @@
 logger = logging.getLogger(__name__)
 
 
-def _is_too_many_requests_error(exception: BaseException) -> bool:
-    """Check if exception is a 429 error."""
-    return (
-        isinstance(exception, httpx.HTTPStatusError)
-        and exception.response.status_code == 429
-    )
+def _is_retryable_server_error(exception: BaseException) -> bool:
+    """Check if exception is a retryable HTTP error (429 or transient 5xx).
+
+    Only 502 Bad Gateway, 503 Service Unavailable, and 504 Gateway Timeout
+    are retried. 500 Internal Server Error is excluded as it may indicate
+    permanent server bugs.
+
+    Args:
+        exception: The exception to check.
+
+    Returns:
+        True if the exception is a retryable HTTP status error.
+    """
+    if not isinstance(exception, httpx.HTTPStatusError):
+        return False
+    status = exception.response.status_code
+    return status in (429, 502, 503, 504)
 
 
 class APIClient:
@@ -59,10 +70,11 @@ def __init__(
         retry_decorator = self._create_retry_decorator()
         self._standard_query_with_retry = retry_decorator(self._standard_query)
         self._streaming_query_with_retry = retry_decorator(self._streaming_query)
+        self._rlsapi_infer_query_with_retry = retry_decorator(self._rlsapi_infer_query)
 
     def _create_retry_decorator(self) -> Any:
         return retry(
-            retry=retry_if_exception(_is_too_many_requests_error),
+            retry=retry_if_exception(_is_retryable_server_error),
             stop=stop_after_attempt(
                 self.config.num_retries + 1
             ),  # +1 to account for the initial attempt
@@ -186,6 +198,8 @@ def query(
 
             if self.config.endpoint_type == "streaming":
                 response = self._streaming_query_with_retry(api_request)
+            elif self.config.endpoint_type == "infer":
+                response = self._rlsapi_infer_query_with_retry(api_request)
             else:
                 response = self._standard_query_with_retry(api_request)
 
@@ -196,7 +210,7 @@ def query(
         except RetryError as e:
             raise APIError(
                 f"Maximum retry attempts ({self.config.num_retries}) reached "
-                "due to persistent rate limiting (HTTP 429)."
+                "due to retryable server errors (HTTP 429/5xx)."
             ) from e
 
     def _prepare_request(
@@ -285,8 +299,7 @@ def _standard_query(self, api_request: APIRequest) -> APIResponse:
         except httpx.TimeoutException as e:
             raise self._handle_timeout_error("standard", self.config.timeout) from e
         except httpx.HTTPStatusError as e:
-            # Re-raise 429 errors without conversion to allow retry decorator to handle them
-            if e.response.status_code == 429:
+            if _is_retryable_server_error(e):
                 raise
             raise self._handle_http_error(e) from e
         except ValueError as e:
@@ -313,8 +326,7 @@ def _streaming_query(self, api_request: APIRequest) -> APIResponse:
         except httpx.TimeoutException as e:
             raise self._handle_timeout_error("streaming", self.config.timeout) from e
         except httpx.HTTPStatusError as e:
-            # Re-raise 429 errors without conversion to allow retry decorator to handle them
-            if e.response.status_code == 429:
+            if _is_retryable_server_error(e):
                 raise
             raise self._handle_http_error(e) from e
         except ValueError as e:
@@ -324,6 +336,167 @@ def _streaming_query(self, api_request: APIRequest) -> APIResponse:
         except Exception as e:
             raise self._handle_unexpected_error(e, "streaming query") from e
 
+    def _build_infer_request(self, api_request: APIRequest) -> dict[str, object]:
+        """Build request payload for the RLSAPI /infer endpoint.
+
+        Converts the standard APIRequest into the infer-specific format,
+        renaming 'query' to 'question' and adding 'include_metadata'.
+
+        Args:
+            api_request: The prepared API request.
+
+        Returns:
+            Dictionary suitable for posting to the /infer endpoint.
+        """
+        request_data = api_request.model_dump(exclude_none=True)
+        # `extra_request_params` are not forwarded to `/infer` — the
+        # endpoint only accepts `question` and `include_metadata`.
+        infer_request: dict[str, object] = {
+            "question": request_data.pop("query"),
+            "include_metadata": True,
+        }
+        logger.debug(
+            "RLSAPI infer request URL: /api/lightspeed/%s/infer",
+            self.config.version,
+        )
+        logger.debug(
+            "RLSAPI infer request: version=%s, include_metadata=%s, "
+            "question_length=%d",
+            self.config.version,
+            True,
+            len(str(infer_request.get("question", ""))),
+        )
+        return infer_request
+
+    def _extract_infer_data(self, response_data: dict[str, Any]) -> None:
+        """Extract and promote fields from the infer response 'data' envelope.
+
+        Mutates response_data in place, mapping nested data fields
+        (text, request_id, tokens, tool_calls, tool_results) to the
+        top-level keys expected by APIResponse.from_raw_response().
+
+        Args:
+            response_data: The raw JSON response dict to update.
+        """
+        if "data" not in response_data:
+            return
+        data = response_data["data"]
+        if "text" in data:
+            response_data["response"] = data["text"]
+        if "request_id" in data:
+            response_data["conversation_id"] = data["request_id"]
+        if "input_tokens" in data:
+            response_data["input_tokens"] = data["input_tokens"]
+        if "output_tokens" in data:
+            response_data["output_tokens"] = data["output_tokens"]
+        if "tool_calls" in data:
+            response_data["tool_calls"] = data["tool_calls"]
+        if "tool_results" in data:
+            tool_results = data["tool_results"]
+            rag_chunks: list[dict[str, str]] = []
+            for result in tool_results:
+                if result.get("type") == "mcp_call":
+                    content = result.get("content")
+                    if not isinstance(content, str) or not content:
+                        logger.warning(
+                            "Skipping mcp_call tool result with missing or "
+                            "non-string content: %r",
+                            content,
+                        )
+                        continue
+                    rag_chunks.extend(
+                        [{"content": chunk} for chunk in content.split("---")]
+                    )
+            response_data["rag_chunks"] = rag_chunks
+
+    def _format_infer_tool_calls(self, response_data: dict[str, Any]) -> None:
+        """Format infer endpoint tool calls to standard list[list[dict]] format.
+
+        Converts raw tool_call dicts (with 'name'/'args' keys) to the
+        normalised format used by APIResponse, and matches each call to
+        its corresponding tool_result by ID when available.
+
+        Args:
+            response_data: The raw JSON response dict to update in place.
+        """
+        if "tool_calls" not in response_data or not response_data["tool_calls"]:
+            return
+        raw_tool_calls = response_data["tool_calls"]
+        formatted_tool_calls = []
+        for tool_call in raw_tool_calls:
+            if isinstance(tool_call, dict):
+                formatted_tool: dict[str, object] = {
+                    "tool_name": tool_call.get("name", ""),
+                    "arguments": tool_call.get("args", {}),
+                }
+                if "tool_results" in response_data.get("data", {}):
+                    tool_call_id = tool_call.get("id")
+                    matching_result = next(
+                        (
+                            r
+                            for r in response_data["data"]["tool_results"]
+                            if r.get("id") == tool_call_id
+                        ),
+                        None,
+                    )
+                    if matching_result:
+                        formatted_tool["result"] = matching_result.get(
+                            "content", matching_result.get("status", "")
+                        )
+                        formatted_tool["status"] = matching_result.get("status", "")
+                formatted_tool_calls.append([formatted_tool])
+        response_data["tool_calls"] = formatted_tool_calls
+
+    def _rlsapi_infer_query(self, api_request: APIRequest) -> APIResponse:
+        """Query the RLSAPI /infer endpoint for tool call and RAG metadata.
+
+        The infer endpoint uses a different request/response format than
+        the standard query/streaming endpoints, converting "query" to
+        "question" and parsing tool_calls and rag_chunks from tool_results.
+
+        Args:
+            api_request: The prepared API request.
+
+        Returns:
+            APIResponse with response text, tool calls, and RAG contexts.
+
+        Raises:
+            APIError: If the request fails or response is invalid.
+        """
+        if not self.client:
+            raise APIError("HTTP client not initialized")
+        try:
+            infer_request = self._build_infer_request(api_request)
+
+            response = self.client.post(
+                f"/api/lightspeed/{self.config.version}/infer",
+                json=infer_request,
+            )
+            response.raise_for_status()
+
+            response_data = response.json()
+            self._extract_infer_data(response_data)
+
+            if "response" not in response_data:
+                raise APIError("API response missing 'response' field")
+
+            self._format_infer_tool_calls(response_data)
+
+            return APIResponse.from_raw_response(response_data)
+
+        except httpx.TimeoutException as e:
+            raise self._handle_timeout_error("infer", self.config.timeout) from e
+        except httpx.HTTPStatusError as e:
+            if _is_retryable_server_error(e):
+                raise
+            raise self._handle_http_error(e) from e
+        except ValueError as e:
+            raise self._handle_validation_error(e) from e
+        except APIError:
+            raise
+        except Exception as e:
+            raise self._handle_unexpected_error(e, "infer query") from e
+
     def _handle_response_errors(self, response: httpx.Response) -> None:
         """Handle HTTP response errors for streaming endpoint."""
         if response.status_code != 200:

src/lightspeed_evaluation/core/constants.py

@@ -56,7 +56,7 @@
 DEFAULT_API_VERSION = "v1"
 DEFAULT_API_TIMEOUT = 300
 DEFAULT_ENDPOINT_TYPE = "streaming"
-SUPPORTED_ENDPOINT_TYPES = ["streaming", "query"]
+SUPPORTED_ENDPOINT_TYPES = ["streaming", "query", "infer"]
 DEFAULT_API_CACHE_DIR = ".caches/api_cache"
 
 DEFAULT_API_NUM_RETRIES = 3

src/lightspeed_evaluation/core/models/data.py

@@ -370,6 +370,14 @@ class EvaluationData(BaseModel):
         min_length=1,
         description="Tag for grouping and filtering conversations",
     )
+    skip: bool = Field(
+        default=False,
+        description="Skip this conversation during evaluation",
+    )
+    skip_reason: Optional[str] = Field(
+        default=None,
+        description="Why this conversation is skipped (documentation only)",
+    )
 
     # Conversation-level metrics
     conversation_metrics: Optional[list[str]] = Field(

src/lightspeed_evaluation/core/models/system.py

@@ -272,7 +272,7 @@ class APIConfig(BaseModel):
     )
     endpoint_type: str = Field(
         default=DEFAULT_ENDPOINT_TYPE,
-        description="API endpoint type (streaming or query)",
+        description="API endpoint type (streaming, query, or infer)",
     )
     timeout: int = Field(
         default=DEFAULT_API_TIMEOUT, ge=1, description="Request timeout in seconds"
@@ -302,7 +302,7 @@ class APIConfig(BaseModel):
         ge=0,
         description=(
             "Maximum number of retry attempts for API calls on "
-            "429 Too Many Requests errors"
+            "retryable server errors (HTTP 429/5xx)"
         ),
     )