feat: Add per-request LLM token usage metrics (#117)

* feat: Record LLM token metrics via Prometheus

Wire up the existing token metrics infrastructure to actually record
token usage from LLM calls. The MetricsCollector already had
record_tokens() method and Prometheus counters (hindsight.tokens.input,
hindsight.tokens.output), but they were never being populated.

Changes:
- Import get_metrics_collector in llm_wrapper.py
- Call record_tokens() after successful LLM calls for:
  - OpenAI/Groq (using response.usage.prompt_tokens, completion_tokens)
  - Anthropic (using response.usage.input_tokens, output_tokens)
  - Gemini (using response.usage_metadata.prompt_token_count, candidates_token_count)
- Add test file to verify token metrics are recorded

Note: Ollama's native API doesn't return token usage, so metrics
are not recorded for that provider.

The token metrics will now be available via /metrics endpoint:
- hindsight_tokens_input_total
- hindsight_tokens_output_total

* feat: add per-request token usage tracking to retain and reflect endpoints

- Add TokenUsage model with input_tokens, output_tokens, total_tokens
- Return usage metrics in retain response (sync operations only)
- Return usage metrics in reflect response
- Update Python, TypeScript, and Rust clients
- Add API documentation for usage fields
- Add changelog entry

Author: omototo

hindsight-api/hindsight_api/api/http.py

@@ -36,7 +36,7 @@ def _parse_metadata(metadata: Any) -> dict[str, Any]:
 from hindsight_api import MemoryEngine
 from hindsight_api.engine.db_utils import acquire_with_retry
 from hindsight_api.engine.memory_engine import Budget, fq_table
-from hindsight_api.engine.response_models import VALID_RECALL_FACT_TYPES
+from hindsight_api.engine.response_models import VALID_RECALL_FACT_TYPES, TokenUsage
 from hindsight_api.extensions import HttpExtension, OperationValidationError, load_extension
 from hindsight_api.metrics import create_metrics_collector, get_metrics_collector, initialize_metrics
 from hindsight_api.models import RequestContext
@@ -364,7 +364,15 @@ class RetainResponse(BaseModel):
 
     model_config = ConfigDict(
         populate_by_name=True,
-        json_schema_extra={"example": {"success": True, "bank_id": "user123", "items_count": 2, "async": False}},
+        json_schema_extra={
+            "example": {
+                "success": True,
+                "bank_id": "user123",
+                "items_count": 2,
+                "async": False,
+                "usage": {"input_tokens": 500, "output_tokens": 100, "total_tokens": 600},
+            }
+        },
     )
 
     success: bool
@@ -373,6 +381,10 @@ class RetainResponse(BaseModel):
     is_async: bool = Field(
         alias="async", serialization_alias="async", description="Whether the operation was processed asynchronously"
     )
+    usage: TokenUsage | None = Field(
+        default=None,
+        description="Token usage metrics for LLM calls during fact extraction (only present for synchronous operations)",
+    )
 
 
 class FactsIncludeOptions(BaseModel):
@@ -472,6 +484,7 @@ class ReflectResponse(BaseModel):
                     "summary": "AI is transformative",
                     "key_points": ["Used in healthcare", "Discussed recently"],
                 },
+                "usage": {"input_tokens": 1500, "output_tokens": 500, "total_tokens": 2000},
             }
         }
     )
@@ -482,6 +495,10 @@ class ReflectResponse(BaseModel):
         default=None,
         description="Structured output parsed according to the request's response_schema. Only present when response_schema was provided in the request.",
     )
+    usage: TokenUsage | None = Field(
+        default=None,
+        description="Token usage metrics for LLM calls during reflection.",
+    )
 
 
 class BanksResponse(BaseModel):
@@ -1290,6 +1307,7 @@ async def api_reflect(
                 text=core_result.text,
                 based_on=based_on_facts,
                 structured_output=core_result.structured_output,
+                usage=core_result.usage,
             )
 
         except OperationValidationError as e:
@@ -2016,12 +2034,12 @@ async def api_retain(
             else:
                 # Synchronous processing: wait for completion (record metrics)
                 with metrics.record_operation("retain", bank_id=bank_id):
-                    result = await app.state.memory.retain_batch_async(
-                        bank_id=bank_id, contents=contents, request_context=request_context
+                    result, usage = await app.state.memory.retain_batch_async(
+                        bank_id=bank_id, contents=contents, request_context=request_context, return_usage=True
                     )
 
                 return RetainResponse.model_validate(
-                    {"success": True, "bank_id": bank_id, "items_count": len(contents), "async": False}
+                    {"success": True, "bank_id": bank_id, "items_count": len(contents), "async": False, "usage": usage}
                 )
         except OperationValidationError as e:
             raise HTTPException(status_code=e.status_code, detail=e.reason)

hindsight-api/hindsight_api/engine/llm_wrapper.py

@@ -23,6 +23,8 @@
     ENV_LLM_MAX_CONCURRENT,
     ENV_LLM_TIMEOUT,
 )
+from ..metrics import get_metrics_collector
+from .response_models import TokenUsage
 
 # Seed applied to every Groq request for deterministic behavior.
 DEFAULT_LLM_SEED = 4242
@@ -174,6 +176,7 @@ async def call(
         max_backoff: float = 60.0,
         skip_validation: bool = False,
         strict_schema: bool = False,
+        return_usage: bool = False,
     ) -> Any:
         """
         Make an LLM API call with retry logic.
@@ -189,9 +192,11 @@ async def call(
             max_backoff: Maximum backoff time in seconds.
             skip_validation: Return raw JSON without Pydantic validation.
             strict_schema: Use strict JSON schema enforcement (OpenAI only). Guarantees all required fields.
+            return_usage: If True, return tuple (result, TokenUsage) instead of just result.
 
         Returns:
-            Parsed response if response_format is provided, otherwise text content.
+            If return_usage=False: Parsed response if response_format is provided, otherwise text content.
+            If return_usage=True: Tuple of (result, TokenUsage) with token counts from the LLM call.
 
         Raises:
             OutputTooLongError: If output exceeds token limits.
@@ -203,7 +208,14 @@ async def call(
             # Handle Gemini provider separately
             if self.provider == "gemini":
                 return await self._call_gemini(
-                    messages, response_format, max_retries, initial_backoff, max_backoff, skip_validation, start_time
+                    messages,
+                    response_format,
+                    max_retries,
+                    initial_backoff,
+                    max_backoff,
+                    skip_validation,
+                    start_time,
+                    return_usage,
                 )
 
             # Handle Anthropic provider separately
@@ -217,6 +229,7 @@ async def call(
                     max_backoff,
                     skip_validation,
                     start_time,
+                    return_usage,
                 )
 
             # Handle Ollama with native API for structured output (better schema enforcement)
@@ -231,6 +244,7 @@ async def call(
                     max_backoff,
                     skip_validation,
                     start_time,
+                    return_usage,
                 )
 
             call_params = {
@@ -379,21 +393,41 @@ async def call(
                         response = await self._client.chat.completions.create(**call_params)
                         result = response.choices[0].message.content
 
-                    # Log slow calls
+                    # Record token usage metrics
                     duration = time.time() - start_time
                     usage = response.usage
-                    if duration > 10.0:
-                        ratio = max(1, usage.completion_tokens) / usage.prompt_tokens
+                    input_tokens = usage.prompt_tokens or 0 if usage else 0
+                    output_tokens = usage.completion_tokens or 0 if usage else 0
+                    total_tokens = usage.total_tokens or 0 if usage else 0
+
+                    if usage:
+                        get_metrics_collector().record_tokens(
+                            operation=scope,
+                            bank_id="llm",
+                            input_tokens=input_tokens,
+                            output_tokens=output_tokens,
+                        )
+
+                    # Log slow calls
+                    if duration > 10.0 and usage:
+                        ratio = max(1, output_tokens) / max(1, input_tokens)
                         cached_tokens = 0
                         if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details:
                             cached_tokens = getattr(usage.prompt_tokens_details, "cached_tokens", 0) or 0
                         cache_info = f", cached_tokens={cached_tokens}" if cached_tokens > 0 else ""
                         logger.info(
                             f"slow llm call: model={self.provider}/{self.model}, "
-                            f"input_tokens={usage.prompt_tokens}, output_tokens={usage.completion_tokens}, "
-                            f"total_tokens={usage.total_tokens}{cache_info}, time={duration:.3f}s, ratio out/in={ratio:.2f}"
+                            f"input_tokens={input_tokens}, output_tokens={output_tokens}, "
+                            f"total_tokens={total_tokens}{cache_info}, time={duration:.3f}s, ratio out/in={ratio:.2f}"
                         )
 
+                    if return_usage:
+                        token_usage = TokenUsage(
+                            input_tokens=input_tokens,
+                            output_tokens=output_tokens,
+                            total_tokens=total_tokens,
+                        )
+                        return result, token_usage
                     return result
 
                 except LengthFinishReasonError as e:
@@ -452,6 +486,7 @@ async def _call_anthropic(
         max_backoff: float,
         skip_validation: bool,
         start_time: float,
+        return_usage: bool = False,
     ) -> Any:
         """Handle Anthropic-specific API calls."""
         from anthropic import APIConnectionError, APIStatusError, RateLimitError
@@ -524,17 +559,35 @@ async def _call_anthropic(
                 else:
                     result = content
 
-                # Log slow calls
+                # Record token usage metrics
                 duration = time.time() - start_time
-                if duration > 10.0:
-                    input_tokens = response.usage.input_tokens
-                    output_tokens = response.usage.output_tokens
+                input_tokens = response.usage.input_tokens or 0 if response.usage else 0
+                output_tokens = response.usage.output_tokens or 0 if response.usage else 0
+                total_tokens = input_tokens + output_tokens
+
+                if response.usage:
+                    get_metrics_collector().record_tokens(
+                        operation="memory",
+                        bank_id="llm",
+                        input_tokens=input_tokens,
+                        output_tokens=output_tokens,
+                    )
+
+                # Log slow calls
+                if duration > 10.0 and response.usage:
                     logger.info(
                         f"slow llm call: model={self.provider}/{self.model}, "
                         f"input_tokens={input_tokens}, output_tokens={output_tokens}, "
                         f"time={duration:.3f}s"
                     )
 
+                if return_usage:
+                    token_usage = TokenUsage(
+                        input_tokens=input_tokens,
+                        output_tokens=output_tokens,
+                        total_tokens=total_tokens,
+                    )
+                    return result, token_usage
                 return result
 
             except json.JSONDecodeError as e:
@@ -589,6 +642,7 @@ async def _call_ollama_native(
         max_backoff: float,
         skip_validation: bool,
         start_time: float,
+        return_usage: bool = False,
     ) -> Any:
         """
         Call Ollama using native API with JSON schema enforcement.
@@ -663,11 +717,35 @@ async def _call_ollama_native(
                         else:
                             raise
 
+                    # Extract token usage from Ollama response
+                    # Ollama returns prompt_eval_count (input) and eval_count (output)
+                    input_tokens = result.get("prompt_eval_count", 0) or 0
+                    output_tokens = result.get("eval_count", 0) or 0
+                    total_tokens = input_tokens + output_tokens
+
+                    # Record to metrics
+                    if input_tokens > 0 or output_tokens > 0:
+                        get_metrics_collector().record_tokens(
+                            operation="memory",
+                            bank_id="llm",
+                            input_tokens=input_tokens,
+                            output_tokens=output_tokens,
+                        )
+
                     # Validate against Pydantic model or return raw JSON
                     if skip_validation:
-                        return json_data
+                        validated_result = json_data
                     else:
-                        return response_format.model_validate(json_data)
+                        validated_result = response_format.model_validate(json_data)
+
+                    if return_usage:
+                        token_usage = TokenUsage(
+                            input_tokens=input_tokens,
+                            output_tokens=output_tokens,
+                            total_tokens=total_tokens,
+                        )
+                        return validated_result, token_usage
+                    return validated_result
 
                 except httpx.HTTPStatusError as e:
                     last_exception = e
@@ -710,6 +788,7 @@ async def _call_gemini(
         max_backoff: float,
         skip_validation: bool,
         start_time: float,
+        return_usage: bool = False,
     ) -> Any:
         """Handle Gemini-specific API calls."""
         # Convert OpenAI-style messages to Gemini format
@@ -786,16 +865,36 @@ async def _call_gemini(
                 else:
                     result = content
 
-                # Log slow calls
+                # Record token usage metrics
                 duration = time.time() - start_time
-                if duration > 10.0 and hasattr(response, "usage_metadata") and response.usage_metadata:
+                input_tokens = 0
+                output_tokens = 0
+                if hasattr(response, "usage_metadata") and response.usage_metadata:
                     usage = response.usage_metadata
-                    logger.info(
-                        f"slow llm call: model={self.provider}/{self.model}, "
-                        f"input_tokens={usage.prompt_token_count}, output_tokens={usage.candidates_token_count}, "
-                        f"time={duration:.3f}s"
+                    input_tokens = usage.prompt_token_count or 0
+                    output_tokens = usage.candidates_token_count or 0
+                    get_metrics_collector().record_tokens(
+                        operation="memory",
+                        bank_id="llm",
+                        input_tokens=input_tokens,
+                        output_tokens=output_tokens,
                     )
 
+                    # Log slow calls
+                    if duration > 10.0:
+                        logger.info(
+                            f"slow llm call: model={self.provider}/{self.model}, "
+                            f"input_tokens={input_tokens}, output_tokens={output_tokens}, "
+                            f"time={duration:.3f}s"
+                        )
+
+                if return_usage:
+                    token_usage = TokenUsage(
+                        input_tokens=input_tokens,
+                        output_tokens=output_tokens,
+                        total_tokens=input_tokens + output_tokens,
+                    )
+                    return result, token_usage
                 return result
 
             except json.JSONDecodeError as e: