feat: bake sampling_rate into Judge at construction; simplify Evaluator to List[Judge] (#159)

Author: jsonbailey

packages/sdk/server-ai/src/ldai/client.py

@@ -306,14 +306,24 @@ def create_judge(
                         print('Relevance score:', relevance_eval.score)
         """
         self._client.track(_TRACK_USAGE_CREATE_JUDGE, context, key, 1)
+        return self._create_judge_instance(key, context, default, variables, default_ai_provider)
 
-        try:
-            if variables:
-                if 'message_history' in variables:
-                    pass
-                if 'response_to_evaluate' in variables:
-                    pass
+    def _create_judge_instance(
+        self,
+        key: str,
+        context: Context,
+        default: Optional[AIJudgeConfigDefault] = None,
+        variables: Optional[Dict[str, Any]] = None,
+        default_ai_provider: Optional[str] = None,
+        sample_rate: float = 1.0,
+    ) -> Optional[Judge]:
+        """
+        Construct a Judge for ``key`` without emitting the public create-judge usage event.
 
+        Used both by the public :meth:`create_judge` and by :meth:`_build_evaluator`
+        when materializing judges referenced by an AI config's judge configuration.
+        """
+        try:
             extended_variables = dict(variables) if variables else {}
             extended_variables['message_history'] = '{{message_history}}'
             extended_variables['response_to_evaluate'] = '{{response_to_evaluate}}'
@@ -329,45 +339,11 @@ def create_judge(
             if not provider:
                 return None
 
-            return Judge(judge_config, provider)
-        except Exception as error:
+            return Judge(judge_config, provider, sample_rate=sample_rate)
+        except Exception as e:
+            log.warning('Failed to initialize judge %r: %s', key, e)
             return None
 
-    def _initialize_judges(
-        self,
-        judge_configs: List[JudgeConfiguration.Judge],
-        context: Context,
-        variables: Optional[Dict[str, Any]] = None,
-        default_ai_provider: Optional[str] = None,
-    ) -> Dict[str, Judge]:
-        """
-        Initialize judges from judge configurations.
-
-        :param judge_configs: List of judge configurations
-        :param context: Standard Context used when evaluating flags
-        :param variables: Dictionary of values for instruction interpolation
-        :param default_ai_provider: Optional default AI provider to use
-        :return: Dictionary of judge instances keyed by their configuration keys
-        """
-        judges: Dict[str, Judge] = {}
-
-        for judge_config in judge_configs:
-            try:
-                judge = self.create_judge(
-                    judge_config.key,
-                    context,
-                    AIJudgeConfigDefault.disabled(),
-                    variables,
-                    default_ai_provider,
-                )
-                if judge:
-                    judges[judge_config.key] = judge
-            except Exception as e:
-                log.warning(f'Failed to initialize judge {judge_config.key!r}: {e}')
-                continue
-
-        return judges
-
     def _build_evaluator(
         self,
         judge_configuration: Optional[JudgeConfiguration],
@@ -387,11 +363,19 @@ def _build_evaluator(
         """
         if not judge_configuration or not judge_configuration.judges:
             return Evaluator.noop()
-        judges = self._initialize_judges(
-            judge_configuration.judges, context, default_ai_provider=default_ai_provider,
-            variables=variables,
-        )
-        return Evaluator(judges, judge_configuration)
+        judge_instances: List[Judge] = []
+        for jc in judge_configuration.judges:
+            judge = self._create_judge_instance(
+                jc.key,
+                context,
+                AIJudgeConfigDefault.disabled(),
+                variables,
+                default_ai_provider,
+                sample_rate=jc.sampling_rate,
+            )
+            if judge is not None:
+                judge_instances.append(judge)
+        return Evaluator(judge_instances)
 
     async def create_model(
         self,

packages/sdk/server-ai/src/ldai/evaluator.py

@@ -3,11 +3,10 @@
 from __future__ import annotations
 
 import asyncio
-from typing import Dict, List
+from typing import List
 
 from ldai import log
 from ldai.judge import Judge
-from ldai.models import JudgeConfiguration
 from ldai.providers.types import JudgeResult
 
 
@@ -20,19 +19,18 @@ class Evaluator:
     not need to construct this directly.
     """
 
-    def __init__(self, judges: Dict[str, Judge], judge_configuration: JudgeConfiguration):
+    def __init__(self, judges: List[Judge]):
         """
         Initialize the Evaluator.
 
-        :param judges: Mapping of judge config key to initialized Judge instances
-        :param judge_configuration: The judge configuration specifying which judges to run
+        :param judges: List of initialized Judge instances. Each Judge already
+            carries its own ``sample_rate`` set at construction time.
         """
         self._judges = judges
-        self._judge_configuration = judge_configuration
 
     @classmethod
     def noop(cls) -> Evaluator:
-        return cls({}, JudgeConfiguration(judges=[]))
+        return cls([])
 
     def evaluate(
         self,
@@ -62,16 +60,12 @@ async def _run_judges(
 
         :param input_text: The input that was provided to the AI model
         :param output_text: The AI-generated output to evaluate
-        :return: List of JudgeResult instances (one per configured judge that was found)
+        :return: List of JudgeResult instances (one per configured judge)
         """
-        if not self._judge_configuration.judges:
+        if not self._judges:
             log.debug('No judges configured, no evaluations to run')
             return []
         results: List[JudgeResult] = []
-        for jc in self._judge_configuration.judges:
-            judge = self._judges.get(jc.key)
-            if not judge:
-                log.warning(f'Judge not enabled: {jc.key}')
-                continue
-            results.append(await judge.evaluate(input_text, output_text, jc.sampling_rate))
+        for judge in self._judges:
+            results.append(await judge.evaluate(input_text, output_text))
         return results

packages/sdk/server-ai/src/ldai/judge/__init__.py

@@ -24,31 +24,37 @@ def __init__(
         self,
         ai_config: AIJudgeConfig,
         model_runner: ModelRunner,
+        sample_rate: float = 1.0,
     ):
         """
         Initialize the Judge.
 
         :param ai_config: The judge AI configuration
         :param model_runner: The model runner to use for evaluation
+        :param sample_rate: Default sampling rate (0-1) used when ``evaluate``
+            is called without an explicit ``sampling_rate`` (defaults to 1).
         """
         self._ai_config = ai_config
         self._model_runner = model_runner
+        self.sample_rate = sample_rate
         self._evaluation_response_structure = EvaluationSchemaBuilder.build()
 
     async def evaluate(
         self,
         input_text: str,
         output_text: str,
-        sampling_rate: float = 1.0,
+        sampling_rate: Optional[float] = None,
     ) -> JudgeResult:
         """
         Evaluates an AI response using the judge's configuration.
 
         :param input_text: The input prompt or question that was provided to the AI
         :param output_text: The AI-generated response to be evaluated
-        :param sampling_rate: Sampling rate (0-1) to determine if evaluation should be processed (defaults to 1)
+        :param sampling_rate: Sampling rate (0-1) to determine if evaluation should be processed.
+            When ``None`` (the default), falls back to ``self.sample_rate``.
         :return: The result of the judge evaluation.
         """
+        effective_rate = sampling_rate if sampling_rate is not None else self.sample_rate
         judge_result = JudgeResult(judge_config_key=self._ai_config.key)
 
         try:
@@ -64,8 +70,8 @@ async def evaluate(
                 judge_result.error_message = 'Judge configuration must include messages'
                 return judge_result
 
-            if random.random() > sampling_rate:
-                log.debug(f'Judge evaluation skipped due to sampling rate: {sampling_rate}')
+            if random.random() > effective_rate:
+                log.debug(f'Judge evaluation skipped due to sampling rate: {effective_rate}')
                 return judge_result
 
             judge_result.sampled = True
@@ -100,14 +106,15 @@ async def evaluate_messages(
         self,
         messages: list[LDMessage],
         response: ModelResponse,
-        sampling_ratio: float = 1.0,
+        sampling_ratio: Optional[float] = None,
     ) -> JudgeResult:
         """
         Evaluates an AI response from chat messages and response.
 
         :param messages: Array of messages representing the conversation history
         :param response: The AI response to be evaluated
-        :param sampling_ratio: Sampling ratio (0-1) to determine if evaluation should be processed (defaults to 1)
+        :param sampling_ratio: Sampling ratio (0-1) to determine if evaluation should be processed.
+            When ``None`` (the default), falls back to ``self.sample_rate``.
         :return: The result of the judge evaluation.
         """
         input_text = '\r\n'.join([msg.content for msg in messages]) if messages else ''

packages/sdk/server-ai/tests/test_judge.py

@@ -121,6 +121,20 @@ def test_judge_initializes_with_evaluation_metric_key(
         assert 'score' in judge._evaluation_response_structure['properties']
         assert 'reasoning' in judge._evaluation_response_structure['properties']
 
+    def test_judge_sample_rate_defaults_to_one(
+        self, judge_config_with_key: AIJudgeConfig, mock_runner
+    ):
+        """sample_rate should default to 1.0 when not provided."""
+        judge = Judge(judge_config_with_key, mock_runner)
+        assert judge.sample_rate == 1.0
+
+    def test_judge_sample_rate_can_be_set(
+        self, judge_config_with_key: AIJudgeConfig, mock_runner
+    ):
+        """sample_rate should be settable via the constructor."""
+        judge = Judge(judge_config_with_key, mock_runner, sample_rate=0.25)
+        assert judge.sample_rate == 0.25
+
 
 class TestJudgeEvaluate:
     """Tests for Judge.evaluate() method."""
@@ -308,6 +322,32 @@ async def test_evaluate_respects_sampling_rate(
         assert result.success is False
         mock_runner.invoke_structured_model.assert_not_called()
 
+    @pytest.mark.asyncio
+    async def test_evaluate_uses_instance_sample_rate_when_arg_omitted(
+        self, judge_config_with_key: AIJudgeConfig, mock_runner
+    ):
+        """When sampling_rate arg is omitted, the instance's sample_rate is used."""
+        judge = Judge(judge_config_with_key, mock_runner, sample_rate=0.0)
+
+        result = await judge.evaluate("input", "output")
+
+        assert isinstance(result, JudgeResult)
+        assert result.sampled is False
+        mock_runner.invoke_structured_model.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_evaluate_arg_overrides_instance_sample_rate(
+        self, judge_config_with_key: AIJudgeConfig, mock_runner
+    ):
+        """An explicit sampling_rate=0.0 must override an instance sample_rate of 1.0."""
+        judge = Judge(judge_config_with_key, mock_runner, sample_rate=1.0)
+
+        result = await judge.evaluate("input", "output", sampling_rate=0.0)
+
+        assert isinstance(result, JudgeResult)
+        assert result.sampled is False
+        mock_runner.invoke_structured_model.assert_not_called()
+
 
 class TestJudgeEvaluateMessages:
     """Tests for Judge.evaluate_messages() method."""