diff --git a/README.md b/README.md index f5e40b87..2ab0f416 100644 --- a/README.md +++ b/README.md @@ -217,8 +217,10 @@ uv run lightspeed-eval --system-config --eval-data new canonical identifier. +DEPRECATED_METRIC_NAMES: dict[str, str] = { + "ragas:context_precision_with_reference": "ragas:context_precision", + "ragas:context_precision_without_reference": "ragas:context_utilization", +} diff --git a/src/lightspeed_evaluation/core/metrics/ragas.py b/src/lightspeed_evaluation/core/metrics/ragas.py index 9ba22cbd..92c04a19 100644 --- a/src/lightspeed_evaluation/core/metrics/ragas.py +++ b/src/lightspeed_evaluation/core/metrics/ragas.py @@ -1,6 +1,7 @@ """Ragas metrics evaluation using LLM Manager with Ragas 0.4+ API.""" import errno +import logging import math import threading from typing import Any, Optional @@ -28,6 +29,13 @@ from lightspeed_evaluation.core.models import EvaluationScope, TurnData from lightspeed_evaluation.core.system.exceptions import EvaluationError +logger = logging.getLogger(__name__) + +_DEPRECATED_RAGAS_METRICS: dict[str, str] = { + "context_precision_with_reference": "context_precision", + "context_precision_without_reference": "context_utilization", +} + def _clamp_score(score: float) -> float: """Clamp score to [0, 1] range to handle floating-point precision issues. @@ -88,10 +96,11 @@ def __init__(self, llm_manager: LLMManager, embedding_manager: EmbeddingManager) # Context/Retrieval evaluation metrics "context_recall": self._evaluate_context_recall, "context_relevance": self._evaluate_context_relevance, - "context_precision_with_reference": self._evaluate_context_precision_with_reference, - "context_precision_without_reference": ( - self._evaluate_context_precision_without_reference - ), + "context_precision": self._evaluate_context_precision, + "context_utilization": self._evaluate_context_utilization, + # Deprecated aliases (backward compatibility) + "context_precision_with_reference": self._evaluate_context_precision, + "context_precision_without_reference": self._evaluate_context_utilization, } @property @@ -133,6 +142,14 @@ def evaluate( scope: EvaluationScope, ) -> tuple[Optional[float], str]: """Evaluate a Ragas metric.""" + if metric_name in _DEPRECATED_RAGAS_METRICS: + new_name = _DEPRECATED_RAGAS_METRICS[metric_name] + logger.warning( + "Metric 'ragas:%s' is deprecated, use 'ragas:%s' instead.", + metric_name, + new_name, + ) + if metric_name not in self.supported_metrics: return None, f"Unsupported Ragas metric: {metric_name}" @@ -220,16 +237,16 @@ def _evaluate_faithfulness( score = _clamp_score(float(result.value)) return score, f"Ragas faithfulness: {score:.2f}" - def _evaluate_context_precision_without_reference( + def _evaluate_context_utilization( self, _conv_data: Any, _turn_idx: Optional[int], turn_data: Optional[TurnData], is_conversation: bool, ) -> tuple[Optional[float], str]: - """Evaluate context precision without reference using Ragas 0.4+ score().""" + """Evaluate context utilization using Ragas 0.4+ ContextUtilization.""" if is_conversation: - return None, "Context precision without reference is a turn-level metric" + return None, "Context utilization is a turn-level metric" query, response, contexts = self._extract_turn_data(turn_data) @@ -242,21 +259,21 @@ def _evaluate_context_precision_without_reference( ) score = _clamp_score(float(result.value)) - return score, f"Ragas context precision without reference: {score:.2f}" + return score, f"Ragas context utilization: {score:.2f}" - def _evaluate_context_precision_with_reference( + def _evaluate_context_precision( self, _conv_data: Any, _turn_idx: Optional[int], turn_data: Optional[TurnData], is_conversation: bool, ) -> tuple[Optional[float], str]: - """Evaluate context precision with reference using Ragas 0.4+ score().""" + """Evaluate context precision using Ragas 0.4+ ContextPrecision.""" if is_conversation: - return None, "Context precision with reference is a turn-level metric" + return None, "Context precision is a turn-level metric" if turn_data is None: - return None, "TurnData is required for context precision with reference" + return None, "TurnData is required for context precision" query, _, contexts = self._extract_turn_data(turn_data) @@ -269,7 +286,7 @@ def _evaluate_context_precision_with_reference( ) score = _clamp_score(float(result.value)) - return score, f"Ragas context precision with reference: {score:.2f}" + return score, f"Ragas context precision: {score:.2f}" def _evaluate_context_recall( self, diff --git a/src/lightspeed_evaluation/core/system/validator.py b/src/lightspeed_evaluation/core/system/validator.py index 950139e6..7b5e0ae4 100644 --- a/src/lightspeed_evaluation/core/system/validator.py +++ b/src/lightspeed_evaluation/core/system/validator.py @@ -7,6 +7,7 @@ import yaml from pydantic import ValidationError +from lightspeed_evaluation.core.constants import DEPRECATED_METRIC_NAMES from lightspeed_evaluation.core.models import EvaluationData, TurnData from lightspeed_evaluation.core.models.data import DatasetMetadata from lightspeed_evaluation.core.system.exceptions import DataValidationError @@ -32,6 +33,15 @@ "required_fields": ["response", "contexts"], "description": "requires 'response' and 'contexts' fields", }, + "ragas:context_precision": { + "required_fields": ["response", "contexts", "expected_response"], + "description": "requires 'response', 'contexts', and 'expected_response' fields", + }, + "ragas:context_utilization": { + "required_fields": ["response", "contexts"], + "description": "requires 'response' and 'contexts' fields", + }, + # Deprecated aliases (backward compatibility) "ragas:context_precision_with_reference": { "required_fields": ["response", "contexts", "expected_response"], "description": "requires 'response', 'contexts', and 'expected_response' fields", @@ -455,7 +465,11 @@ def _validate_metrics_availability(self, data: EvaluationData) -> None: for turn_data in data.turns: if turn_data.turn_metrics: for metric in turn_data.turn_metrics: - if metric not in self._turn_level_metrics: + canonical = DEPRECATED_METRIC_NAMES.get(metric, metric) + if ( + metric not in self._turn_level_metrics + and canonical not in self._turn_level_metrics + ): turn_data.add_invalid_metric(metric) self.validation_errors.append( f"Conversation {conversation_id}, Turn {turn_data.turn_id}: " @@ -465,7 +479,11 @@ def _validate_metrics_availability(self, data: EvaluationData) -> None: # Validate conversation metrics if data.conversation_metrics: for metric in data.conversation_metrics: - if metric not in self._conversation_level_metrics: + canonical = DEPRECATED_METRIC_NAMES.get(metric, metric) + if ( + metric not in self._conversation_level_metrics + and canonical not in self._conversation_level_metrics + ): data.add_invalid_metric(metric) self.validation_errors.append( f"Conversation {conversation_id}: Unknown conversation metric '{metric}'"