feat: add quota monitoring metrics

Signed-off-by: Major Hayden <major@redhat.com>

Author: major

src/app/endpoints/responses.py

@@ -4,6 +4,7 @@
 
 import asyncio
 import json
+import time
 from collections.abc import AsyncIterator
 from datetime import UTC, datetime
 from typing import Annotated, Any, Final, Optional, cast
@@ -39,6 +40,7 @@
 from configuration import configuration
 from constants import SUBSTITUTED_INSTRUCTIONS_PLACEHOLDER
 from log import get_logger
+from metrics import recording
 from models.config import Action
 from models.requests import ResponsesRequest
 from models.responses import (
@@ -136,6 +138,27 @@ def _get_user_agent(request: Request) -> Optional[str]:
     return sanitized or None
 
 
+def _check_response_quota(user_id: str, endpoint_path: str) -> None:
+    """Check response quota availability and record bounded quota metrics."""
+    quota_start_time = time.monotonic()
+    try:
+        check_tokens_available(configuration.quota_limiters, user_id)
+    except HTTPException:
+        recording.record_quota_check(
+            endpoint_path,
+            "user_id",
+            "failure",
+            time.monotonic() - quota_start_time,
+        )
+        raise
+    recording.record_quota_check(
+        endpoint_path,
+        "user_id",
+        "success",
+        time.monotonic() - quota_start_time,
+    )
+
+
 responses_response: dict[int | str, dict[str, Any]] = {
     200: ResponsesResponse.openapi_response(),
     401: UnauthorizedResponse.openapi_response(
@@ -275,11 +298,12 @@ async def responses_endpoint_handler(
     started_at = datetime.now(UTC)
     rh_identity_context = get_rh_identity_context(request)
     user_id, _, _, token = auth
+    endpoint_path = "/v1/responses"
 
     await check_mcp_auth(configuration, mcp_headers, token, request.headers)
 
     # Check token availability
-    check_tokens_available(configuration.quota_limiters, user_id)
+    _check_response_quota(user_id, endpoint_path)
 
     # Enforce RBAC: optionally disallow overriding model in requests
     validate_model_provider_override(
@@ -331,7 +355,6 @@ async def responses_endpoint_handler(
     )
     attachments_text = extract_attachments_text(original_request.input)
 
-    endpoint_path = "/v1/responses"
     moderation_result = await run_shield_moderation(
         client,
         input_text + "\n\n" + attachments_text,

src/app/endpoints/rlsapi_v1.py

@@ -532,6 +532,36 @@ def _resolve_quota_subject(request: Request, auth: AuthTuple) -> Optional[str]:
     return system_id
 
 
+def _check_infer_quota(
+    request: Request, auth: AuthTuple, endpoint_path: str
+) -> Optional[str]:
+    """Check infer quota availability and record bounded quota metrics."""
+    quota_id = _resolve_quota_subject(request, auth)
+    quota_type = configuration.rlsapi_v1.quota_subject or "disabled"
+    if quota_id is None:
+        recording.record_quota_check(endpoint_path, quota_type, "skipped", 0.0)
+        return None
+
+    quota_start_time = time.monotonic()
+    try:
+        check_tokens_available(configuration.quota_limiters, quota_id)
+    except HTTPException:
+        recording.record_quota_check(
+            endpoint_path,
+            quota_type,
+            "failure",
+            time.monotonic() - quota_start_time,
+        )
+        raise
+    recording.record_quota_check(
+        endpoint_path,
+        quota_type,
+        "success",
+        time.monotonic() - quota_start_time,
+    )
+    return quota_id
+
+
 def _build_infer_response(
     response_text: str,
     request_id: str,
@@ -669,12 +699,11 @@ async def infer_endpoint(  # pylint: disable=R0914
     """
     # Authentication enforced by get_auth_dependency(), authorization by @authorize decorator.
     check_configuration_loaded(configuration)
+    endpoint_path = "/v1/infer"
 
     # Quota enforcement: resolve subject and check availability before any work.
     # No-op when quota_subject is not configured or no quota limiters exist.
-    quota_id = _resolve_quota_subject(request, auth)
-    if quota_id is not None:
-        check_tokens_available(configuration.quota_limiters, quota_id)
+    quota_id = _check_infer_quota(request, auth, endpoint_path)
 
     request_id = get_suid()
 
@@ -685,8 +714,6 @@ async def infer_endpoint(  # pylint: disable=R0914
         "Request %s: Combined input source length: %d", request_id, len(input_source)
     )
 
-    endpoint_path = "/v1/infer"
-
     # Run shield moderation on user input before inference.
     # Uses all configured shields; no-op when no shields are registered.
     # Runs before model/tool discovery so blocked requests short-circuit

src/metrics/__init__.py

@@ -55,3 +55,17 @@
     "LLM tokens received",
     ["provider", "model", "endpoint"],
 )
+
+# Counter to track pre-request quota checks by bounded quota category.
+quota_checks_total = Counter(
+    "ls_quota_checks_total",
+    "Quota availability checks",
+    ["endpoint", "quota_type", "result"],
+)
+
+# Histogram to measure quota availability check latency by bounded quota category.
+quota_check_duration_seconds = Histogram(
+    "ls_quota_check_duration_seconds",
+    "Quota availability check duration",
+    ["endpoint", "quota_type", "result"],
+)

src/metrics/recording.py

@@ -109,3 +109,23 @@ def record_llm_token_usage(
         )
     except (AttributeError, TypeError, ValueError):
         logger.warning("Failed to update token metrics", exc_info=True)
+
+
+def record_quota_check(
+    endpoint_path: str, quota_type: str, result: str, duration: float
+) -> None:
+    """Record a quota availability check.
+
+    Args:
+        endpoint_path: API endpoint path for metric labeling.
+        quota_type: Bounded quota category, not the subject identifier.
+        result: Bounded result label, such as ``success``, ``skipped``, or ``failure``.
+        duration: Quota check duration in seconds.
+    """
+    try:
+        metrics.quota_checks_total.labels(endpoint_path, quota_type, result).inc()
+        metrics.quota_check_duration_seconds.labels(
+            endpoint_path, quota_type, result
+        ).observe(duration)
+    except (AttributeError, TypeError, ValueError):
+        logger.warning("Failed to update quota check metrics", exc_info=True)

tests/unit/metrics/test_recording.py

@@ -1,5 +1,8 @@
 """Unit tests for Prometheus metric recording helpers."""
 
+from typing import Any
+
+import pytest
 from pytest_mock import MockerFixture
 
 from metrics import recording
@@ -159,3 +162,42 @@ def test_record_llm_token_usage_logs_metric_errors(mocker: MockerFixture) -> Non
     mock_logger.warning.assert_called_once_with(
         "Failed to update token metrics", exc_info=True
     )
+
+
+@pytest.fixture(name="recording_logger")
+def recording_logger_fixture(mocker: MockerFixture) -> Any:
+    """Patch the metric recording logger for failure assertions."""
+    return mocker.patch("metrics.recording.logger")
+
+
+@pytest.mark.parametrize("failing_metric", ["counter", "histogram"])
+def test_record_quota_check_updates_metrics_and_logs_errors(
+    mocker: MockerFixture,
+    recording_logger: Any,
+    failing_metric: str,
+) -> None:
+    """Test quota helper counter and histogram updates plus both failure points."""
+    mock_counter = mocker.patch("metrics.recording.metrics.quota_checks_total")
+    mock_histogram = mocker.patch(
+        "metrics.recording.metrics.quota_check_duration_seconds"
+    )
+
+    recording.record_quota_check("/v1/infer", "org_id", "success", 0.75)
+
+    mock_counter.labels.assert_called_once_with("/v1/infer", "org_id", "success")
+    mock_counter.labels.return_value.inc.assert_called_once()
+    mock_histogram.labels.assert_called_once_with("/v1/infer", "org_id", "success")
+    mock_histogram.labels.return_value.observe.assert_called_once_with(0.75)
+
+    mock_counter.reset_mock()
+    mock_histogram.reset_mock()
+    if failing_metric == "counter":
+        mock_counter.labels.return_value.inc.side_effect = TypeError("bad")
+    else:
+        mock_histogram.labels.return_value.observe.side_effect = TypeError("bad")
+
+    recording.record_quota_check("/v1/infer", "org_id", "failure", 0.75)
+
+    recording_logger.warning.assert_called_once_with(
+        "Failed to update quota check metrics", exc_info=True
+    )