API latency calculation

Author: xmican10

README.md

@@ -14,6 +14,7 @@ A comprehensive framework for evaluating GenAI applications.
 - **API Integration**: Direct integration with external API for real-time data generation (if enabled)
 - **Setup/Cleanup Scripts**: Support for running setup and cleanup scripts before/after each conversation evaluation (applicable when API is enabled)
 - **Token Usage Tracking**: Track input/output tokens for both API calls and Judge LLM evaluations (per-judge tracking for panel mode)
+- **API Latency Tracking**: Measure and analyze API response times with percentile statistics (p50, p95, p99) for performance monitoring
 - **Streaming Performance Metrics**: Capture time-to-first-token (TTFT), streaming duration, and tokens/second when using streaming endpoint
 - **Statistical Analysis**: Statistics for every metric with score distribution analysis
 - **Rich Output**: CSV, JSON, TXT reports + visualization graphs (pass rates, distributions, heatmaps)

config/system.yaml

@@ -274,6 +274,7 @@ storage:
       - "response"
       - "api_input_tokens"
       - "api_output_tokens"
+      - "agent_latency"
       # Streaming performance metrics (only populated when using streaming endpoint)
       - "time_to_first_token"    # Time to first token in seconds
       - "streaming_duration"      # Total streaming duration in seconds

docs/EVALUATION_GUIDE.md

@@ -1127,6 +1127,7 @@ Contains every metric evaluation with:
 - Detailed reasoning
 - Query and response text
 - Execution time
+- API latency
 
 **Use for:** Drilling into specific failures, detailed analysis
 
@@ -1180,6 +1181,16 @@ ragas:faithfulness:
 - **ERROR** ⚠️: Evaluation couldn't complete (missing data, API failure, etc.)
 - **SKIPPED** ⏭️: Evaluation skipped due to prior failure (when `skip_on_failure` is enabled)
 
+### Performance Metrics (API Enabled Only)
+
+**API Latency**: Response time per API call with percentile stats (p50, p95, p99). Cached responses (zero tokens) are excluded to avoid skewing statistics.
+
+**Streaming Metrics**: Time-to-first-token, streaming duration, and tokens/second when using streaming endpoints.
+
+**Token Usage**: Track consumption across Judge LLM, embeddings, and API calls.
+
+**Note:** Cached responses are detected by zero `api_input_tokens` and `api_output_tokens` — latency is set to 0 for these.
+
 ### Score Quality Levels
 
 | Score | Quality | Recommendation |
@@ -1912,4 +1923,3 @@ This comprehensive guide has covered everything you need to know to effectively
 *This guide is designed to make AI evaluation accessible to everyone. Whether you're a product manager making decisions, a QA engineer testing systems, or a developer integrating evaluation into workflows, you now have everything you need to ensure your AI applications meet quality standards.*
 
 **Happy Evaluating! 🚀**
-

src/lightspeed_evaluation/core/constants.py

@@ -106,6 +106,7 @@
     # Streaming performance metrics
     "time_to_first_token",
     "streaming_duration",
+    "agent_latency",
     "tokens_per_second",
     "tool_calls",
     "contexts",

src/lightspeed_evaluation/core/models/data.py

@@ -84,6 +84,11 @@ class TurnData(StreamingMetricsMixin):
         default=0, ge=0, description="Output tokens used by API call"
     )
 
+    # API execution time tracking (per turn)
+    agent_latency: float = Field(
+        default=0, ge=0, description="API call latency for this turn in seconds"
+    )
+
     # Per-turn metrics support
     turn_metrics: Optional[list[str]] = Field(
         default=None,
@@ -515,6 +520,11 @@ class EvaluationResult(MetricResult, StreamingMetricsMixin):
     execution_time: float = Field(
         default=0, ge=0, description="Execution time in seconds"
     )
+    agent_latency: float = Field(
+        default=0,
+        ge=0,
+        description="API latency in seconds (per turn or average for conversation)",
+    )
     api_input_tokens: int = Field(default=0, ge=0, description="API input tokens used")
     api_output_tokens: int = Field(
         default=0, ge=0, description="API output tokens used"

src/lightspeed_evaluation/core/models/quality.py

@@ -9,7 +9,13 @@
 
 from pydantic import BaseModel, Field
 
-from lightspeed_evaluation.core.models.summary import MetricStats, ScoreStatistics
+from lightspeed_evaluation.core.models.summary import (
+    MetricStats,
+    NumericStats,
+    ScoreStatistics,
+    AgentTokenStats,
+)
+
 
 logger = logging.getLogger(__name__)
 
@@ -44,17 +50,18 @@ class QualityReport(BaseModel):
         default_factory=list,
         description="Warnings about quality metrics configuration or usage",
     )
-    api_latency: float = Field(
-        default=0.0, description="[Placeholder] Average API response time in seconds"
+    agent_latency_stats: Optional[NumericStats] = Field(
+        default=None, description="Agent latency statistics"
     )
-    api_tokens: int = Field(
-        default=0,
-        description="[Placeholder] Total number of tokens consumed across all API calls",
+    agent_token_stats: Optional[AgentTokenStats] = Field(
+        default=None, description="Agent token usage statistics"
     )
 
     @staticmethod
     def create_report(
         by_metric: dict[str, MetricStats],
+        agent_latency_stats: Optional[NumericStats],
+        agent_token_stats: Optional[AgentTokenStats],
         quality_score_metrics: list[str],
     ) -> Optional["QualityReport"]:
         """Creates a quality report with aggregated quality score from selected metrics.
@@ -64,6 +71,8 @@ def create_report(
 
         Args:
             by_metric: Dictionary mapping metric identifiers to their computed statistics.
+            agent_latency_stats: Agent API latency statistics (p50, p95, p99).
+            agent_token_stats: Agent token usage statistics with percentiles.
             quality_score_metrics: Metric identifiers to include in quality score calculation.
                 All specified metrics must exist in by_metric.
 
@@ -148,14 +157,13 @@ def create_report(
                 if stats is not None:
                     extra_metrics[metric_id] = stats
 
-        # Calculate aggregated quality score
-        aggregated_score = QualityReport._calculate_quality_score(quality_metrics)
-
         return QualityReport(
-            quality_score=aggregated_score,
+            quality_score=QualityReport._calculate_quality_score(quality_metrics),
             quality_metrics=quality_metrics,
             extra_metrics=extra_metrics,
             warnings=warnings,
+            agent_latency_stats=agent_latency_stats,
+            agent_token_stats=agent_token_stats,
         )
 
     @staticmethod

src/lightspeed_evaluation/core/models/summary.py

@@ -14,20 +14,23 @@
 from lightspeed_evaluation.core.models.data import EvaluationData, EvaluationResult
 from lightspeed_evaluation.core.output.statistics import (
     bootstrap_intervals,
+    calculate_field_numeric_stats_from_evaluation_data,
     calculate_api_token_usage,
     calculate_streaming_stats,
 )
 
 
 class NumericStats(BaseModel):
-    """Numeric statistics for a set of values (e.g., TTFT, duration)."""
+    """Numeric statistics for a set of values (e.g., TTFT, duration, latency)."""
 
     count: int = Field(default=0, description="Number of values")
     mean: Optional[float] = Field(default=None, description="Mean value")
     median: Optional[float] = Field(default=None, description="Median value")
     std: Optional[float] = Field(default=None, description="Standard deviation")
     min_value: Optional[float] = Field(default=None, description="Minimum value")
     max_value: Optional[float] = Field(default=None, description="Maximum value")
+    p95: Optional[float] = Field(default=None, description="95th percentile")
+    p99: Optional[float] = Field(default=None, description="99th percentile")
 
 
 class ScoreStatistics(BaseModel):
@@ -101,14 +104,28 @@ class StreamingStats(BaseModel):
     )
 
 
-class ApiTokenUsage(BaseModel):
-    """API token usage totals."""
+class AgentTokenStats(BaseModel):
+    """Agent token usage statistics with percentiles."""
+
+    input: Optional[NumericStats] = Field(
+        default=None, description="Input token statistics"
+    )
+    output: Optional[NumericStats] = Field(
+        default=None, description="Output token statistics"
+    )
+
+
+class AgentTokenUsage(BaseModel):
+    """Agent token usage totals and statistics."""
 
     total_api_input_tokens: int = Field(default=0, description="Total API input tokens")
     total_api_output_tokens: int = Field(
         default=0, description="Total API output tokens"
     )
     total_api_tokens: int = Field(default=0, description="Total API tokens")
+    statistics: Optional[AgentTokenStats] = Field(
+        default=None, description="Agent token usage statistics with percentiles"
+    )
 
 
 class EvaluationSummary(BaseModel):
@@ -134,8 +151,11 @@ class EvaluationSummary(BaseModel):
     by_tag: dict[str, TagStats] = Field(
         default_factory=dict, description="Statistics per tag"
     )
-    api_tokens: Optional[ApiTokenUsage] = Field(
-        default=None, description="API token usage (when evaluation data provided)"
+    agent_token_usage: Optional[AgentTokenUsage] = Field(
+        default=None, description="Agent token usage with totals and statistics"
+    )
+    agent_latency_stats: Optional[NumericStats] = Field(
+        default=None, description="API latency statistics (when API enabled)"
     )
     streaming: Optional[StreamingStats] = Field(
         default=None, description="Streaming performance stats (when available)"
@@ -172,11 +192,14 @@ def from_results(
         by_tag = _compute_tag_stats(results, compute_confidence_intervals)
 
         # Compute API token usage and streaming stats if evaluation data provided
-        api_tokens = None
+        agent_token_usage = None
         streaming = None
+        agent_latency_stats = None
         if evaluation_data:
-            api_tokens = _compute_api_token_usage(evaluation_data)
+            agent_token_usage = _compute_agent_token_usage(evaluation_data)
             streaming = _compute_streaming_stats(evaluation_data)
+            # Compute Agent latency statistics from evaluation data
+            agent_latency_stats = _compute_agent_latency_stats(evaluation_data)
 
         return cls(
             timestamp=timestamp,
@@ -185,7 +208,8 @@ def from_results(
             by_metric=by_metric,
             by_conversation=by_conversation,
             by_tag=by_tag,
-            api_tokens=api_tokens,
+            agent_token_usage=agent_token_usage,
+            agent_latency_stats=agent_latency_stats,
             streaming=streaming,
         )
 
@@ -417,7 +441,7 @@ def _numeric_stats_from_dict(raw: dict[str, Any]) -> Optional[NumericStats]:
     """Convert a raw numeric stats dictionary to a NumericStats model.
 
     Args:
-        raw: Dictionary with count, mean, median, std, min, max keys.
+        raw: Dictionary with count, mean, median, std, min, max, p95, p99 keys.
 
     Returns:
         NumericStats instance, or None if count is 0.
@@ -432,28 +456,78 @@ def _numeric_stats_from_dict(raw: dict[str, Any]) -> Optional[NumericStats]:
         std=raw.get("std"),
         min_value=raw.get("min"),
         max_value=raw.get("max"),
+        p95=raw.get("p95"),
+        p99=raw.get("p99"),
     )
 
 
-def _compute_api_token_usage(
+def _compute_agent_token_usage(
     evaluation_data: list[EvaluationData],
-) -> ApiTokenUsage:
-    """Compute API token usage from evaluation data.
+) -> AgentTokenUsage:
+    """Compute agent token usage with totals and statistics from evaluation data.
 
     Args:
         evaluation_data: List of evaluation data with turn-level token counts.
 
     Returns:
-        ApiTokenUsage instance.
+        AgentTokenUsage instance with totals and statistics.
     """
+    stats = _compute_agent_token_stats(evaluation_data)
     raw = calculate_api_token_usage(evaluation_data)
-    return ApiTokenUsage(
+    return AgentTokenUsage(
         total_api_input_tokens=raw["total_api_input_tokens"],
         total_api_output_tokens=raw["total_api_output_tokens"],
         total_api_tokens=raw["total_api_tokens"],
+        statistics=stats,
     )
 
 
+def _compute_agent_latency_stats(
+    evaluation_data: list[EvaluationData],
+) -> Optional[NumericStats]:
+    """Compute agent latency statistics from evaluation data.
+
+    Args:
+        evaluation_data: List of evaluation data containing turn-level latency values.
+
+    Returns:
+        NumericStats instance, or None if no agent latency data available.
+    """
+    if not evaluation_data:
+        return None
+    raw = calculate_field_numeric_stats_from_evaluation_data(
+        evaluation_data, "agent_latency"
+    )
+    return _numeric_stats_from_dict(raw)
+
+
+def _compute_agent_token_stats(
+    evaluation_data: list[EvaluationData],
+) -> Optional[AgentTokenStats]:
+    """Calculate agent token usage statistics with percentiles from evaluation data.
+
+    Args:
+        evaluation_data: List of evaluation data containing turn-level token counts.
+
+    Returns:
+        AgentTokenStats instance with input/output token statistics, or None if no data.
+    """
+    if not evaluation_data:
+        return None
+
+    input_tokens_stats = calculate_field_numeric_stats_from_evaluation_data(
+        evaluation_data, "api_input_tokens"
+    )
+    output_tokens_stats = calculate_field_numeric_stats_from_evaluation_data(
+        evaluation_data, "api_output_tokens"
+    )
+
+    input_stats = _numeric_stats_from_dict(input_tokens_stats)
+    output_stats = _numeric_stats_from_dict(output_tokens_stats)
+
+    return AgentTokenStats(input=input_stats, output=output_stats)
+
+
 def _compute_streaming_stats(
     evaluation_data: list[EvaluationData],
 ) -> Optional[StreamingStats]: