Merge pull request lightspeed-core#187 from asamal4/consistent-config-exception

chore: consistent config exception for config data model val

Author: asamal4

src/lightspeed_evaluation/core/llm/manager.py

@@ -6,6 +6,7 @@
 
 from lightspeed_evaluation.core.models import LLMConfig, SystemConfig
 from lightspeed_evaluation.core.system.env_validator import validate_provider_env
+from lightspeed_evaluation.core.system.exceptions import ConfigurationError
 
 logger = logging.getLogger(__name__)
 
@@ -51,7 +52,7 @@ def __init__(
                     # Create child manager without system_config to avoid recursion
                     judge_manager = LLMManager(resolved_config, judge_id=pool_key)
                     self.judge_managers.append(judge_manager)
-            except ValueError as e:
+            except ConfigurationError as e:
                 logger.error("Failed to resolve judge panel: %s", e)
                 raise
         else:

src/lightspeed_evaluation/core/metrics/geval.py

@@ -26,6 +26,7 @@
 from lightspeed_evaluation.core.llm.deepeval import DeepEvalLLMManager
 from lightspeed_evaluation.core.metrics.manager import MetricLevel, MetricManager
 from lightspeed_evaluation.core.models import GEvalConfig
+from lightspeed_evaluation.core.system.exceptions import ConfigurationError
 
 logger = logging.getLogger(__name__)
 
@@ -111,7 +112,7 @@ def evaluate(  # pylint: disable=R0913,R0917
         # GEvalConfig (criteria, rubrics, threshold, etc.) for evaluation.
         try:
             config = GEvalConfig.from_metadata(raw_config)
-        except (ValueError, ValidationError) as e:
+        except (ValueError, ValidationError, ConfigurationError) as e:
             return None, f"Invalid GEval configuration: {e!s}"
 
         # Convert validated rubrics to DeepEval Rubric objects

src/lightspeed_evaluation/core/models/system.py

@@ -12,7 +12,6 @@
     model_validator,
 )
 
-from lightspeed_evaluation.core.system.exceptions import ConfigurationError
 from lightspeed_evaluation.core.constants import (
     DEFAULT_API_BASE,
     DEFAULT_API_CACHE_DIR,
@@ -45,6 +44,7 @@
     SUPPORTED_GRAPH_TYPES,
     SUPPORTED_OUTPUT_TYPES,
 )
+from lightspeed_evaluation.core.system.exceptions import ConfigurationError
 
 
 class LLMConfig(BaseModel):
@@ -82,7 +82,7 @@ def validate_ssl_cert_file(self) -> "LLMConfig":
 
             # Check if file exists
             if not os.path.isfile(cert_path):
-                raise ValueError(
+                raise ConfigurationError(
                     f"SSL certificate file not found: '{cert_path}'. "
                     f"Original path: '{self.ssl_cert_file}'. "
                     "Please provide a valid path to a CA certificate file "
@@ -519,10 +519,10 @@ def resolve_llm_config(
             Fully resolved LLMConfig
 
         Raises:
-            ValueError: If model_id not found
+            ConfigurationError: If model_id not found
         """
         if model_id not in self.models:
-            raise ValueError(
+            raise ConfigurationError(
                 f"Model '{model_id}' not found in llm_pool.models. "
                 f"Available: {list(self.models.keys())}"
             )
@@ -687,7 +687,7 @@ def validate_rubrics_non_overlapping(self) -> "GEvalConfig":
                     continue
                 # Overlap if not (b < c or d < a)
                 if not (b < c or d < a):
-                    raise ValueError(
+                    raise ConfigurationError(
                         f"Rubric score ranges must not overlap: "
                         f"[{a}, {b}] and [{c}, {d}] overlap"
                     )
@@ -708,9 +708,9 @@ def from_metadata(cls, raw: dict[str, Any]) -> "GEvalConfig":
             ValueError: If raw is not a dict or criteria is missing/empty
                 (only these pre-model_validate checks raise bare ValueError).
             ValidationError: If rubric or config fields fail Pydantic validation:
-                wrong types (e.g. score_range, expected_outcome), invalid structure,
-                or overlapping score ranges (model validator raises ValueError
-                and Pydantic v2 wraps it as ValidationError).
+                wrong types (e.g. score_range, expected_outcome), invalid structure.
+            ConfigurationError: If rubric score ranges overlap (model validator
+                raises ConfigurationError directly, bypassing Pydantic wrapping).
         """
         if not isinstance(raw, dict):
             raise ValueError("GEval config must be a dict")
@@ -801,17 +801,17 @@ def validate_default_metrics_metadata_geval(
 
         Raises:
             ConfigurationError: When a geval:* entry has invalid config (e.g.
-                missing criteria, invalid rubric structure).
-                Re-raised from ValueError or Pydantic ValidationError for a consistent
-                config-failure exception type.
+                missing criteria, invalid rubric structure, overlapping rubrics).
+                Re-raised from ValueError, ValidationError, or ConfigurationError
+                for a consistent config-failure exception type with metric context.
         """
         if not v:
             return v
         for metric_id, meta in v.items():
             if metric_id.startswith("geval:") and isinstance(meta, dict):
                 try:
                     GEvalConfig.from_metadata(meta)
-                except (ValueError, ValidationError) as e:
+                except (ValueError, ValidationError, ConfigurationError) as e:
                     raise ConfigurationError(
                         f"Invalid GEval config for '{metric_id}': {e!s}"
                     ) from e

src/lightspeed_evaluation/runner/evaluation.py

@@ -11,7 +11,10 @@
 
 # Import only lightweight modules at top level
 from lightspeed_evaluation.core.system import ConfigLoader
-from lightspeed_evaluation.core.system.exceptions import DataValidationError
+from lightspeed_evaluation.core.system.exceptions import (
+    ConfigurationError,
+    DataValidationError,
+)
 
 
 def _clear_caches(system_config: SystemConfig) -> None:
@@ -174,7 +177,13 @@ def run_evaluation(  # pylint: disable=too-many-locals
             "SKIPPED": summary["SKIPPED"],
         }
 
-    except (FileNotFoundError, ValueError, RuntimeError, DataValidationError) as e:
+    except (
+        FileNotFoundError,
+        ValueError,
+        RuntimeError,
+        ConfigurationError,
+        DataValidationError,
+    ) as e:
         print(f"\n❌ Evaluation failed: {e}")
         traceback.print_exc()
         return None

tests/unit/core/models/test_system.py

@@ -71,11 +71,11 @@ def test_ssl_cert_file_handling(self, mocker: MockerFixture) -> None:
             os.unlink(cert_path)
 
         # Non-existent file fails
-        with pytest.raises(ValidationError, match="(?i)not found"):
+        with pytest.raises(ConfigurationError, match="(?i)not found"):
             LLMConfig(ssl_cert_file="/tmp/nonexistent_cert_12345.crt")
 
         # Directory fails
-        with pytest.raises(ValidationError):
+        with pytest.raises(ConfigurationError):
             LLMConfig(ssl_cert_file=tempfile.gettempdir())
 
 
@@ -268,7 +268,7 @@ def test_resolve_llm_config(self) -> None:
         assert resolved.timeout == 300
 
         # Unknown model raises error
-        with pytest.raises(ValueError, match="Model 'unknown' not found"):
+        with pytest.raises(ConfigurationError, match="Model 'unknown' not found"):
             pool.resolve_llm_config("unknown")
 
     def test_custom_model_id_and_ssl(self) -> None:
@@ -419,13 +419,13 @@ def test_error_branches(self) -> None:
         with pytest.raises(ConfigurationError, match="llm_pool.*not defined"):
             config.get_judge_configs()
 
-        # get_judge_configs with invalid judge ID raises ValueError
+        # get_judge_configs with invalid judge ID raises error
         pool = LLMPoolConfig(
             models={"gpt-4o-mini": LLMProviderConfig(provider="openai")}
         )
         panel = JudgePanelConfig(judges=["gpt-4o-mini", "nonexistent"])
         config = SystemConfig(llm_pool=pool, judge_panel=panel)
-        with pytest.raises(ValueError, match="Model 'nonexistent' not found"):
+        with pytest.raises(ConfigurationError, match="Model 'nonexistent' not found"):
             config.get_judge_configs()
 
 
@@ -483,13 +483,8 @@ def test_geval_config_rubrics_adjacent_non_overlapping_accepted(self) -> None:
         assert config.rubrics[1].score_range == (4, 7)
 
     def test_geval_config_rubrics_overlapping_fails(self) -> None:
-        """Overlapping rubric ranges fail validation.
-
-        validate_rubrics_non_overlapping raises ValueError, but Pydantic v2
-        wraps it in ValidationError before from_metadata returns, so callers
-        get ValidationError.
-        """
-        with pytest.raises(ValidationError, match="overlap"):
+        """Overlapping rubric ranges fail validation."""
+        with pytest.raises(ConfigurationError, match="overlap"):
             GEvalConfig.from_metadata(
                 {
                     "criteria": "Check.",