Merge pull request lightspeed-core#164 from asamal4/judge-panel-manager

[LEADS-191] feat: handle judge panel config in manager

Author: asamal4

src/lightspeed_evaluation/core/llm/manager.py

@@ -1,11 +1,14 @@
 """LLM Manager - Generic LLM configuration, validation, and parameter provider."""
 
+import logging
 import os
-from typing import Any
+from typing import Any, Optional
 
 from lightspeed_evaluation.core.models import LLMConfig, SystemConfig
 from lightspeed_evaluation.core.system.env_validator import validate_provider_env
 
+logger = logging.getLogger(__name__)
+
 
 class LLMManager:
     """Generic LLM Manager for all use cases (Ragas, DeepEval, Custom metrics).
@@ -14,19 +17,58 @@ class LLMManager:
     - Environment validation for multiple providers
     - Model name construction
     - Provides LLM parameters for consumption by framework-specific managers
+    - Manages judge panel configurations when available
     """
 
-    def __init__(self, config: LLMConfig):
-        """Initialize with validated environment and constructed model name."""
+    def __init__(
+        self,
+        config: LLMConfig,
+        system_config: Optional[SystemConfig] = None,
+    ):
+        """Initialize with validated environment and constructed model name.
+
+        Args:
+            config: Primary LLM configuration (also used as fallback)
+            system_config: Optional full system config for judge panel support
+        """
         self.config = config
-        self.model_name = self._construct_model_name_and_validate()
-        print(
-            f"✅ LLM Manager: {self.config.provider}/{self.config.model} -> {self.model_name}"
-        )
-
-    def _construct_model_name_and_validate(self) -> str:
-        """Construct model name and validate required environment variables."""
-        provider = self.config.provider.lower()
+        self.system_config = system_config
+        self.model_name = self._construct_model_name_and_validate(config)
+
+        # Initialize judge panel if available
+        self.judge_managers: list["LLMManager"] = []
+        if system_config and system_config.judge_panel and system_config.llm_pool:
+            panel = system_config.judge_panel
+            logger.info("Judge panel configured with %d judges", len(panel.judges))
+            # Create LLM managers for each judge using resolved configs from llms pool
+            try:
+                judge_configs = system_config.get_judge_configs()
+                for resolved_config in judge_configs:
+                    # Create child manager without system_config to avoid recursion
+                    judge_manager = LLMManager(resolved_config)
+                    self.judge_managers.append(judge_manager)
+            except ValueError as e:
+                logger.error("Failed to resolve judge panel: %s", e)
+                raise
+        else:
+            # No judge panel - log single LLM info
+            logger.info(
+                "LLM Manager: %s/%s -> %s",
+                self.config.provider,
+                self.config.model,
+                self.model_name,
+            )
+
+    def _construct_model_name_and_validate(self, config: LLMConfig) -> str:
+        """Construct model name and validate required environment variables.
+
+        Args:
+            config: LLM configuration to construct model name for
+
+        Returns:
+            Constructed model name string
+        """
+        provider = config.provider.lower()
 
         # Provider-specific validation and model name construction
         provider_handlers = {
@@ -44,8 +86,8 @@ def _construct_model_name_and_validate(self) -> str:
             return provider_handlers[provider]()
 
         # Generic provider - try as-is with warning
-        print(f"⚠️ Using generic provider format for {provider}")
-        return f"{provider}/{self.config.model}"
+        logger.warning("Using generic provider format for %s", provider)
+        return f"{provider}/{config.model}"
 
     def _handle_hosted_vllm_provider(self) -> str:
         """Handle hosted vLLM provider setup."""
@@ -88,6 +130,59 @@ def _handle_ollama_provider(self) -> str:
         validate_provider_env("ollama")
         return f"ollama/{self.config.model}"
 
+    def has_judge_panel(self) -> bool:
+        """Check if judge panel is configured.
+
+        Returns:
+            True if judge panel is configured (one or more judges)
+        """
+        return len(self.judge_managers) > 0
+
+    def get_judge_managers(self) -> list["LLMManager"]:
+        """Get list of judge LLM managers.
+
+        Returns:
+            List of LLMManager instances. If no panel configured, returns list
+            with single manager (self). Always returns at least one manager.
+        """
+        if self.judge_managers:
+            return self.judge_managers
+        # No panel - return self as single judge
+        return [self]
+
+    def get_primary_judge(self) -> "LLMManager":
+        """Get primary judge LLM manager (first in panel or self).
+
+        This is used when panel is disabled for specific metrics or
+        as fallback when panel is not configured.
+
+        Returns:
+            Primary LLM manager (first judge if panel exists, otherwise self)
+        """
+        if self.judge_managers:
+            return self.judge_managers[0]
+        return self
+
+    def should_use_panel_for_metric(self, metric_identifier: str) -> bool:
+        """Determine if a metric should use judge panel based on enabled_metrics.
+
+        Args:
+            metric_identifier: Metric identifier (e.g., "ragas:faithfulness")
+
+        Returns:
+            True if metric should use judge panel, False otherwise
+        """
+        if self.system_config and self.system_config.judge_panel:
+            enabled_metrics = self.system_config.judge_panel.enabled_metrics
+            # If enabled_metrics is None, all metrics use panel
+            if enabled_metrics is None:
+                return True
+            # Check if this specific metric is in the list
+            return metric_identifier in enabled_metrics
+
+        # Default: Don't use panel (use primary judge only)
+        return False
+
     def get_model_name(self) -> str:
         """Get the constructed model name."""
         return self.model_name
@@ -110,10 +205,24 @@ def get_config(self) -> LLMConfig:
 
     @classmethod
     def from_system_config(cls, system_config: SystemConfig) -> "LLMManager":
-        """Create LLM Manager from system configuration."""
-        return cls(system_config.llm)
+        """Create LLM Manager from system configuration.
+
+        Args:
+            system_config: System configuration with LLM and optional judge panel
+
+        Returns:
+            LLMManager with judge panel support if configured
+        """
+        return cls(system_config.llm, system_config=system_config)
 
     @classmethod
     def from_llm_config(cls, llm_config: LLMConfig) -> "LLMManager":
-        """Create LLM Manager from LLMConfig directly."""
+        """Create LLM Manager from LLMConfig directly (no judge panel support).
+
+        Args:
+            llm_config: LLM configuration
+
+        Returns:
+            LLMManager without judge panel support
+        """
         return cls(llm_config)

tests/unit/core/llm/test_llm_manager.py

@@ -1,9 +1,21 @@
 """Unit tests for LLM Manager."""
 
+import logging
+
 import pytest
 from pytest_mock import MockerFixture
 
-from lightspeed_evaluation.core.models import LLMConfig, SystemConfig
+from lightspeed_evaluation.core.models import (
+    LLMConfig,
+    SystemConfig,
+    LLMPoolConfig,
+    JudgePanelConfig,
+)
+from lightspeed_evaluation.core.models.system import (
+    LLMDefaultsConfig,
+    LLMParametersConfig,
+    LLMProviderConfig,
+)
 from lightspeed_evaluation.core.llm.manager import LLMManager
 
 
@@ -128,7 +140,7 @@ def test_initialization_hosted_vllm(self, mocker: MockerFixture) -> None:
         assert manager.model_name == "hosted_vllm/mistral-7b"
 
     def test_initialization_generic_provider(
-        self, mocker: MockerFixture, capsys: pytest.CaptureFixture
+        self, mocker: MockerFixture, caplog: pytest.LogCaptureFixture
     ) -> None:
         """Test initialization with unknown/generic provider."""
         config = LLMConfig(
@@ -138,14 +150,14 @@ def test_initialization_generic_provider(
         )
         mocker.patch("lightspeed_evaluation.core.llm.manager.validate_provider_env")
 
-        manager = LLMManager(config)
+        with caplog.at_level(logging.WARNING):
+            manager = LLMManager(config)
 
         # Should construct generic model name
         assert manager.model_name == "custom_provider/custom-model"
 
-        # Should print warning
-        captured = capsys.readouterr()
-        assert "generic" in captured.out.lower() or "warning" in captured.out.lower()
+        # Should log warning about generic provider
+        assert any("generic" in record.message.lower() for record in caplog.records)
 
     def test_get_model_name(
         self, basic_llm_config: LLMConfig, mocker: MockerFixture
@@ -231,18 +243,147 @@ def test_llm_params_with_custom_values(self, mocker: MockerFixture) -> None:
         assert params["timeout"] == 120
         assert params["num_retries"] == 5
 
-    def test_initialization_prints_message(
+    def test_initialization_logs_message(
         self,
         basic_llm_config: LLMConfig,
         mocker: MockerFixture,
-        capsys: pytest.CaptureFixture,
+        caplog: pytest.LogCaptureFixture,
     ) -> None:
-        """Test that initialization prints configuration message."""
+        """Test that initialization logs configuration message."""
         mocker.patch("lightspeed_evaluation.core.llm.manager.validate_provider_env")
 
-        LLMManager(basic_llm_config)
+        with caplog.at_level(logging.INFO):
+            LLMManager(basic_llm_config)
+
+        # Should log LLM manager info
+        assert any("LLM Manager" in record.message for record in caplog.records)
+        assert any("openai" in record.message for record in caplog.records)
+        assert any("gpt-4" in record.message for record in caplog.records)
+
+
+def _create_llm_pool_with_judges(
+    judges: list[tuple[str, str]],
+    enabled_metrics: list[str] | None = None,
+) -> tuple[LLMPoolConfig, JudgePanelConfig]:
+    """Helper to create LLMPoolConfig and JudgePanelConfig from judge list.
+
+    Args:
+        judges: List of (provider, model) tuples.
+        enabled_metrics: Optional list of metrics to enable for panel.
+    """
+    models: dict[str, LLMProviderConfig] = {}
+    for provider, model in judges:
+        models[model] = LLMProviderConfig(provider=provider)
+
+    pool = LLMPoolConfig(
+        defaults=LLMDefaultsConfig(
+            parameters=LLMParametersConfig(temperature=0.0, max_completion_tokens=512)
+        ),
+        models=models,
+    )
+    judge_ids = [model for _, model in judges]
+    panel = JudgePanelConfig(judges=judge_ids, enabled_metrics=enabled_metrics)
+    return pool, panel
+
+
+class TestLLMManagerJudgePanel:
+    """Tests for LLMManager judge panel functionality."""
+
+    def test_without_judge_panel(self, mocker: MockerFixture) -> None:
+        """Test LLMManager without judge panel configured."""
+        mocker.patch("lightspeed_evaluation.core.llm.manager.validate_provider_env")
+        manager = LLMManager(LLMConfig(provider="openai", model="gpt-4o-mini"))
+
+        assert not manager.has_judge_panel()
+        assert len(manager.judge_managers) == 0
+        assert len(manager.get_judge_managers()) == 1
+        assert manager.get_primary_judge() is manager
+        assert not manager.should_use_panel_for_metric("ragas:faithfulness")
+
+    def test_with_judge_panel(self, mocker: MockerFixture) -> None:
+        """Test LLMManager with judge panel configured."""
+        mocker.patch("lightspeed_evaluation.core.llm.manager.validate_provider_env")
+
+        pool, panel = _create_llm_pool_with_judges(
+            [
+                ("openai", "gpt-4o-mini"),
+                ("openai", "gpt-4o"),
+                ("gemini", "gemini-2.0-flash-exp"),
+            ]
+        )
+        system_config = SystemConfig(llm_pool=pool, judge_panel=panel)
+        manager = LLMManager.from_system_config(system_config)
+
+        # Panel detected
+        assert manager.has_judge_panel()
+        assert len(manager.judge_managers) == 3
+
+        # Judge managers
+        judges = manager.get_judge_managers()
+        assert len(judges) == 3
+        assert judges[0].config.model == "gpt-4o-mini"
+        assert judges[1].config.model == "gpt-4o"
+        assert judges[2].config.model == "gemini-2.0-flash-exp"
+
+        # Primary judge is first
+        assert manager.get_primary_judge().config.model == "gpt-4o-mini"
+
+    def test_should_use_panel_with_enabled_metrics(self, mocker: MockerFixture) -> None:
+        """Test should_use_panel with enabled_metrics."""
+        mocker.patch("lightspeed_evaluation.core.llm.manager.validate_provider_env")
+
+        pool, panel = _create_llm_pool_with_judges(
+            [("openai", "gpt-4o-mini")],
+            enabled_metrics=["ragas:faithfulness", "custom:answer_correctness"],
+        )
+        system_config = SystemConfig(llm_pool=pool, judge_panel=panel)
+        manager = LLMManager.from_system_config(system_config)
 
-        captured = capsys.readouterr()
-        assert "LLM Manager" in captured.out
-        assert "openai" in captured.out
-        assert "gpt-4" in captured.out
+        # Metric in list - use panel
+        assert manager.should_use_panel_for_metric("ragas:faithfulness")
+        assert manager.should_use_panel_for_metric("custom:answer_correctness")
+
+        # Metric not in list - don't use panel
+        assert not manager.should_use_panel_for_metric("ragas:response_relevancy")
+
+    def test_should_use_panel_with_enabled_metrics_none(
+        self, mocker: MockerFixture
+    ) -> None:
+        """Test should_use_panel when enabled_metrics is None (all metrics)."""
+        mocker.patch("lightspeed_evaluation.core.llm.manager.validate_provider_env")
+
+        # enabled_metrics=None is the default, meaning all metrics use panel
+        pool, panel = _create_llm_pool_with_judges(
+            [("openai", "gpt-4o-mini")],
+            enabled_metrics=None,
+        )
+        system_config = SystemConfig(llm_pool=pool, judge_panel=panel)
+        manager = LLMManager.from_system_config(system_config)
+
+        # All metrics use panel
+        assert manager.should_use_panel_for_metric("ragas:faithfulness")
+        assert manager.should_use_panel_for_metric("custom:answer_correctness")
+        assert manager.should_use_panel_for_metric("deepeval:conversation_completeness")
+
+    def test_judge_panel_logs_message(
+        self, mocker: MockerFixture, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        """Test judge panel initialization logs messages."""
+        mocker.patch("lightspeed_evaluation.core.llm.manager.validate_provider_env")
+
+        pool, panel = _create_llm_pool_with_judges(
+            [
+                ("openai", "gpt-4o-mini"),
+                ("openai", "gpt-4o"),
+            ]
+        )
+        system_config = SystemConfig(llm_pool=pool, judge_panel=panel)
+
+        with caplog.at_level(logging.INFO):
+            LLMManager.from_system_config(system_config)
+
+        # Should log judge panel info
+        assert any(
+            "Judge panel" in record.message and "2 judges" in record.message
+            for record in caplog.records
+        )