Execution time and evaluation latency impl

Author: xmican10

config/system.yaml

@@ -270,6 +270,7 @@ storage:
       - "threshold"
       - "reason"
       - "execution_time"
+      - "evaluation_latency"
       - "query"
       - "response"
       - "api_input_tokens"

src/lightspeed_evaluation/core/constants.py

@@ -100,6 +100,7 @@
     "query",
     "response",
     "execution_time",
+    "evaluation_latency",
     "api_input_tokens",
     "api_output_tokens",
     "judge_llm_input_tokens",

src/lightspeed_evaluation/core/models/data.py

@@ -529,10 +529,13 @@ class EvaluationResult(MetricResult, StreamingMetricsMixin):
     query: str = Field(default="", description="Query text")
     response: str = Field(default="", description="Response text")
     execution_time: float = Field(
-        default=0, ge=0, description="Execution time in seconds"
+        default=0.0, ge=0, description="Execution time for entire turn in seconds"
+    )
+    evaluation_latency: float = Field(
+        default=0.0, ge=0, description="Evaluation latency in seconds"
     )
     agent_latency: float = Field(
-        default=0,
+        default=0.0,
         ge=0,
         description="API latency in seconds (per turn or average for conversation)",
     )

src/lightspeed_evaluation/core/output/generator.py

@@ -269,11 +269,8 @@ def _generate_csv_report(
                 for column in csv_columns:
                     if hasattr(result, column):
                         value = getattr(result, column)
-                        # Special formatting for execution_time
-                        if column == "execution_time" and value is not None:
-                            row_data.append(f"{value:.3f}")
                         # Convert judge_scores to JSON string
-                        elif column == "judge_scores" and value is not None:
+                        if column == "judge_scores" and value is not None:
                             row_data.append(
                                 json.dumps(
                                     [js.model_dump() for js in value], default=str
@@ -822,7 +819,7 @@ def _result_to_json_dict(r: EvaluationResult) -> dict[str, Any]:
         "result": r.result,
         "score": r.score,
         "threshold": r.threshold,
-        "execution_time": round(r.execution_time, 3),
+        "evaluation_latency": r.evaluation_latency,
         "judge_llm_input_tokens": r.judge_llm_input_tokens,
         "judge_llm_output_tokens": r.judge_llm_output_tokens,
         "judge_scores": (

src/lightspeed_evaluation/core/storage/sql_storage.py

@@ -57,7 +57,7 @@ class EvaluationResultDB(Base):  # pylint: disable=too-few-public-methods
     reason = Column(Text, nullable=True)
     query = Column(Text, nullable=True)
     response = Column(Text, nullable=True)
-    execution_time = Column(Float, nullable=True)
+    evaluation_latency = Column(Float, nullable=True)
     api_input_tokens = Column(Integer, nullable=True)
     api_output_tokens = Column(Integer, nullable=True)
     judge_llm_input_tokens = Column(Integer, nullable=True)
@@ -318,7 +318,7 @@ def _result_to_db_record(self, result: EvaluationResult) -> EvaluationResultDB:
             reason=result.reason,
             query=result.query,
             response=result.response,
-            execution_time=result.execution_time,
+            evaluation_latency=result.evaluation_latency,
             api_input_tokens=result.api_input_tokens,
             api_output_tokens=result.api_output_tokens,
             judge_llm_input_tokens=result.judge_llm_input_tokens,

src/lightspeed_evaluation/pipeline/evaluation/evaluator.py

@@ -86,6 +86,11 @@ def _calculate_agent_latency_per_request(request: EvaluationRequest) -> float:
     return sum(latencies) / len(latencies) if latencies else 0.0
 
 
+def _measure_latency(start_time: float) -> float:
+    """Calculate evaluation latency given start time."""
+    return time.perf_counter() - start_time
+
+
 class MetricsEvaluator:
     """Handles individual metric evaluation with proper scoring and status determination."""
 
@@ -155,7 +160,7 @@ def evaluate_metric(  # pylint: disable=too-many-locals
             EvaluationResult with score, result, token usage, and execution time,
             or None if metric should be skipped (e.g., script metrics when API disabled).
         """
-        start_time = time.time()
+        start_time = time.perf_counter()
 
         try:
             # Create logging summary
@@ -184,9 +189,8 @@ def evaluate_metric(  # pylint: disable=too-many-locals
 
             # Route to appropriate handler
             if framework not in self.handlers:
-                execution_time = time.time() - start_time
                 return self._create_error_result(
-                    request, f"Unsupported framework: {framework}", execution_time
+                    request, f"Unsupported framework: {framework}", start_time
                 )
 
             # Check required data for metric (after API call); skip with ERROR if missing
@@ -198,11 +202,10 @@ def evaluate_metric(  # pylint: disable=too-many-locals
                     request.turn_data, request.metric_identifier
                 )
                 if not ok:
-                    execution_time = time.time() - start_time
                     logger.warning(
                         "Skipping metric due to missing required data: %s", msg
                     )
-                    return self._create_error_result(request, msg, execution_time)
+                    return self._create_error_result(request, msg, start_time)
 
             # Create evaluation scope
             evaluation_scope = EvaluationScope(
@@ -224,7 +227,7 @@ def evaluate_metric(  # pylint: disable=too-many-locals
             # Evaluate metric
             metric_result = self._evaluate_wrapper(request, evaluation_scope, threshold)
 
-            execution_time = time.time() - start_time
+            evaluation_latency = _measure_latency(start_time)
 
             turn_data = request.turn_data
             api_input_tokens, api_output_tokens = (
@@ -240,8 +243,9 @@ def evaluate_metric(  # pylint: disable=too-many-locals
                 metric_metadata=self._extract_metadata_for_csv(request),
                 query=turn_data.query if turn_data else "",
                 response=turn_data.response or "" if turn_data else "",
-                execution_time=execution_time,
+                evaluation_latency=evaluation_latency,
                 agent_latency=agent_latency,
+                execution_time=evaluation_latency + agent_latency,
                 api_input_tokens=api_input_tokens,
                 api_output_tokens=api_output_tokens,
                 # Streaming performance metrics
@@ -266,9 +270,8 @@ def evaluate_metric(  # pylint: disable=too-many-locals
 
         except EvaluationError as e:
             # Any evaluation error should result in ERROR status
-            execution_time = time.time() - start_time
             return self._create_error_result(
-                request, f"Evaluation error: {e}", execution_time
+                request, f"Evaluation error: {e}", start_time
             )
 
     def _will_use_panel(self, metric_identifier: str) -> bool:
@@ -720,14 +723,15 @@ def _evaluate_non_llm(
         )
 
     def _create_error_result(
-        self, request: EvaluationRequest, reason: str, execution_time: float
+        self, request: EvaluationRequest, reason: str, start_time: float
     ) -> EvaluationResult:
         """Create an ERROR result for failed evaluation."""
         turn_data = request.turn_data
         api_input_tokens, api_output_tokens = _calculate_api_token_counts_per_request(
             request
         )
         agent_latency = _calculate_agent_latency_per_request(request)
+        evaluation_latency = _measure_latency(start_time)
         return EvaluationResult(
             conversation_group_id=request.conv_data.conversation_group_id,
             tag=request.conv_data.tag,
@@ -740,8 +744,9 @@ def _create_error_result(
             reason=reason,
             query=turn_data.query if turn_data else "",
             response=turn_data.response or "" if turn_data else "",
-            execution_time=execution_time,
+            evaluation_latency=evaluation_latency,
             agent_latency=agent_latency,
+            execution_time=evaluation_latency + agent_latency,
             api_input_tokens=api_input_tokens,
             api_output_tokens=api_output_tokens,
             # Streaming performance metrics

tests/script/conftest.py

@@ -29,7 +29,7 @@ def sample_evaluation_data() -> tuple[list[dict], list[dict]]:
             "result": "PASS",
             "score": 0.8,
             "threshold": 0.7,
-            "execution_time": 1.0,
+            "evaluation_latency": 1.0,
         },
         {
             "conversation_group_id": "conv1",
@@ -38,7 +38,7 @@ def sample_evaluation_data() -> tuple[list[dict], list[dict]]:
             "result": "PASS",
             "score": 0.9,
             "threshold": 0.7,
-            "execution_time": 1.2,
+            "evaluation_latency": 1.2,
         },
     ]
 
@@ -50,7 +50,7 @@ def sample_evaluation_data() -> tuple[list[dict], list[dict]]:
             "result": "PASS",
             "score": 0.85,
             "threshold": 0.7,
-            "execution_time": 1.1,
+            "evaluation_latency": 1.1,
         },
         {
             "conversation_group_id": "conv1",
@@ -59,7 +59,7 @@ def sample_evaluation_data() -> tuple[list[dict], list[dict]]:
             "result": "FAIL",
             "score": 0.6,
             "threshold": 0.7,
-            "execution_time": 1.0,
+            "evaluation_latency": 1.0,
         },
     ]
 
@@ -198,7 +198,7 @@ def sample_evaluation_summary() -> dict[str, Any]:
                 "result": "PASS",
                 "score": 0.95,
                 "threshold": 0.8,
-                "execution_time": 1.0,
+                "evaluation_latency": 1.0,
             },
             {
                 "conversation_group_id": "conv1",
@@ -207,7 +207,7 @@ def sample_evaluation_summary() -> dict[str, Any]:
                 "result": "PASS",
                 "score": 0.85,
                 "threshold": 0.7,
-                "execution_time": 1.2,
+                "evaluation_latency": 1.2,
             },
         ]
         * 5,  # Repeat to get 10 results

tests/unit/core/models/test_data.py

@@ -469,7 +469,7 @@ def test_default_values(self) -> None:
         assert result.tag == "eval"
         assert result.score is None
         assert result.reason == ""
-        assert result.execution_time == 0
+        assert result.evaluation_latency == 0
 
     def test_explicit_tag_value(self) -> None:
         """Test EvaluationResult with explicit tag value."""
@@ -507,16 +507,16 @@ def test_invalid_result_status_rejected(self) -> None:
                 threshold=0.7,
             )
 
-    def test_negative_execution_time_rejected(self) -> None:
-        """Test that negative execution_time is rejected."""
+    def test_negative_evaluation_latency_rejected(self) -> None:
+        """Test that negative evaluation_latency is rejected."""
         with pytest.raises(ValidationError):
             EvaluationResult(
                 conversation_group_id="conv1",
                 turn_id="turn1",
                 metric_identifier="metric1",
                 result="PASS",
                 threshold=0.7,
-                execution_time=-1,
+                evaluation_latency=-1,
             )
 
     def test_conversation_level_metric_allows_none_turn_id(self) -> None:

tests/unit/core/models/test_summary.py

@@ -25,7 +25,7 @@
     "score": 0.85,
     "threshold": 0.7,
     "reason": "Good",
-    "execution_time": 1.0,
+    "evaluation_latency": 1.0,
     "judge_llm_input_tokens": 100,
     "judge_llm_output_tokens": 50,
 }

tests/unit/core/output/test_generator.py

@@ -323,7 +323,7 @@ def test_generate_csv_with_specific_results(self, tmp_path: Path) -> None:
                 reason="Score is 0.8",
                 query="What is OpenShift?",
                 response="OpenShift is a container platform.",
-                execution_time=1.5,
+                evaluation_latency=1.5,
             ),
             EvaluationResult(
                 conversation_group_id="test_conv",
@@ -335,7 +335,7 @@ def test_generate_csv_with_specific_results(self, tmp_path: Path) -> None:
                 reason="Poor performance",
                 query="How to deploy?",
                 response="Use oc apply.",
-                execution_time=0.8,
+                evaluation_latency=0.8,
                 expected_response="Use oc apply -f deployment.yaml",
             ),
             EvaluationResult(
@@ -348,7 +348,7 @@ def test_generate_csv_with_specific_results(self, tmp_path: Path) -> None:
                 reason="API connection failed",
                 query="Create namespace",
                 response="",
-                execution_time=0.0,
+                evaluation_latency=0.0,
             ),
         ]