fix: build judge input as string; strip legacy message_history/response_to_evaluate messages

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: jsonbailey

packages/ai-providers/server-ai-langchain/src/ldai_langchain/langchain_agent_runner.py

@@ -33,7 +33,7 @@ def __init__(self, agent: Any):
 
     async def run(
         self,
-        input: Any,
+        input: str,
         output_type: Optional[Dict[str, Any]] = None,
     ) -> RunnerResult:
         """
@@ -42,7 +42,7 @@ async def run(
         Delegates to the compiled LangChain agent, which handles
         the tool-calling loop internally.
 
-        :param input: The user prompt or input to the agent
+        :param input: The user prompt string to the agent
         :param output_type: Reserved for future structured output support;
             currently ignored.
         :return: :class:`RunnerResult` with ``content``, ``raw`` response, and

packages/ai-providers/server-ai-langchain/src/ldai_langchain/langchain_model_runner.py

@@ -37,37 +37,25 @@ def get_llm(self) -> BaseChatModel:
 
     async def run(
         self,
-        input: Any,
+        input: str,
         output_type: Optional[Dict[str, Any]] = None,
     ) -> RunnerResult:
         """
         Run the LangChain model with the given input.
 
-        :param input: A string prompt or a list of :class:`LDMessage` objects
+        :param input: A string prompt
         :param output_type: Optional JSON schema dict requesting structured output.
             When provided, ``parsed`` on the returned :class:`RunnerResult` is
             populated with the parsed JSON document.
         :return: :class:`RunnerResult` containing ``content``, ``metrics``,
             ``raw`` and (when ``output_type`` is set) ``parsed``.
         """
-        messages = self._coerce_input(input)
+        messages = [LDMessage(role='user', content=input)]
 
         if output_type is not None:
             return await self._run_structured(messages, output_type)
         return await self._run_completion(messages)
 
-    # convert_messages_to_langchain only accepts List[LDMessage]; _coerce_input
-    # normalizes a bare string to [LDMessage(role='user', ...)] before that step.
-    @staticmethod
-    def _coerce_input(input: Any) -> List[LDMessage]:
-        if isinstance(input, str):
-            return [LDMessage(role='user', content=input)]
-        if isinstance(input, list):
-            return input
-        raise TypeError(
-            f"Unsupported input type for LangChainModelRunner.run: {type(input).__name__}"
-        )
-
     async def _run_completion(self, messages: List[LDMessage]) -> RunnerResult:
         try:
             langchain_messages = convert_messages_to_langchain(messages)

packages/ai-providers/server-ai-openai/src/ldai_openai/openai_agent_runner.py

@@ -47,7 +47,7 @@ def __init__(
 
     async def run(
         self,
-        input: Any,
+        input: str,
         output_type: Optional[Dict[str, Any]] = None,
     ) -> RunnerResult:
         """
@@ -56,7 +56,7 @@ async def run(
         Delegates to the OpenAI Agents SDK ``Runner.run``, which handles the
         tool-calling loop internally.
 
-        :param input: The user prompt or input to the agent
+        :param input: The user prompt string to the agent
         :param output_type: Reserved for future structured output support;
             currently ignored.
         :return: :class:`RunnerResult` with ``content``, ``raw`` response, and

packages/ai-providers/server-ai-openai/src/ldai_openai/openai_model_runner.py

@@ -35,39 +35,25 @@ def __init__(
 
     async def run(
         self,
-        input: Any,
+        input: str,
         output_type: Optional[Dict[str, Any]] = None,
     ) -> RunnerResult:
         """
         Run the OpenAI model with the given input.
 
-        :param input: A string prompt or a list of :class:`LDMessage` objects
+        :param input: A string prompt
         :param output_type: Optional JSON schema dict requesting structured output.
             When provided, ``parsed`` on the returned :class:`RunnerResult` is
             populated with the parsed JSON document.
         :return: :class:`RunnerResult` containing ``content``, ``metrics``,
             ``raw`` and (when ``output_type`` is set) ``parsed``.
         """
-        try:
-            messages = self._coerce_input(input)
-        except TypeError as error:
-            log.warning(f'OpenAI model runner received unsupported input type: {error}')
-            return RunnerResult(content='', metrics=LDAIMetrics(success=False, usage=None))
+        messages = [LDMessage(role='user', content=input)]
 
         if output_type is not None:
             return await self._run_structured(messages, output_type)
         return await self._run_completion(messages)
 
-    @staticmethod
-    def _coerce_input(input: Any) -> List[LDMessage]:
-        if isinstance(input, str):
-            return [LDMessage(role='user', content=input)]
-        if isinstance(input, list):
-            return input
-        raise TypeError(
-            f"Unsupported input type for OpenAIModelRunner.run: {type(input).__name__}"
-        )
-
     async def _run_completion(self, messages: List[LDMessage]) -> RunnerResult:
         try:
             response = await self._client.chat.completions.create(

packages/sdk/server-ai/src/ldai/judge/__init__.py

@@ -1,9 +1,7 @@
 """Judge implementation for AI evaluation."""
 
 import random
-from typing import Any, Dict, Optional, Tuple
-
-import chevron
+from typing import Any, Dict, List, Optional, Tuple
 
 from ldai import log
 from ldai.judge.evaluation_schema_builder import EvaluationSchemaBuilder
@@ -12,6 +10,30 @@
 from ldai.providers.types import JudgeResult, RunnerResult
 
 
+def _strip_legacy_judge_messages(messages: List[LDMessage]) -> List[LDMessage]:
+    """
+    Remove legacy judge template messages from a message list.
+
+    Strips any non-system message whose content contains ``{{message_history}}``
+    or ``{{response_to_evaluate}}``.  These were used by older judge configs to
+    indicate where the SDK should interpolate the evaluated conversation; new
+    configs omit them entirely and rely on the string input built by
+    :meth:`Judge._build_evaluation_input`.
+
+    :param messages: The raw message list from the judge AI config.
+    :return: A new list with legacy template messages removed.
+    """
+    result = []
+    for msg in messages:
+        if msg.role != 'system' and (
+            '{{message_history}}' in msg.content
+            or '{{response_to_evaluate}}' in msg.content
+        ):
+            continue
+        result.append(msg)
+    return result
+
+
 class Judge:
     """
     Judge implementation that handles evaluation functionality and conversation management.
@@ -65,24 +87,19 @@ async def evaluate(
                 judge_result.error_message = 'Judge configuration is missing required evaluationMetricKey'
                 return judge_result
 
-            if not self._ai_config.messages:
-                log.warning('Judge configuration must include messages')
-                judge_result.error_message = 'Judge configuration must include messages'
-                return judge_result
-
             if random.random() > effective_rate:
                 log.debug(f'Judge evaluation skipped due to sampling rate: {effective_rate}')
                 return judge_result
 
             judge_result.sampled = True
 
             tracker = self._ai_config.create_tracker()
-            messages = self._construct_evaluation_messages(input_text, output_text)
+            evaluation_input = self._build_evaluation_input(input_text, output_text)
             assert self._evaluation_response_structure is not None
 
             response = await tracker.track_metrics_of_async(
                 lambda result: result.metrics,
-                lambda: self._model_runner.run(messages, output_type=self._evaluation_response_structure),
+                lambda: self._model_runner.run(evaluation_input, output_type=self._evaluation_response_structure),
             )
 
             if response.parsed is None:
@@ -142,38 +159,21 @@ def get_model_runner(self) -> Runner:
         """
         return self._model_runner
 
-    def _construct_evaluation_messages(self, input_text: str, output_text: str) -> list[LDMessage]:
-        """
-        Constructs evaluation messages by combining judge's config messages with input/output.
-
-        :param input_text: The input text
-        :param output_text: The output text to evaluate
-        :return: List of messages for evaluation
+    def _build_evaluation_input(self, input_text: str, output_text: str) -> str:
         """
-        if not self._ai_config.messages:
-            return []
+        Build the string input for the judge runner.
 
-        messages: list[LDMessage] = []
-        for msg in self._ai_config.messages:
-            # Interpolate message content with reserved variables
-            content = self._interpolate_message(msg.content, {
-                'message_history': input_text,
-                'response_to_evaluate': output_text,
-            })
-            messages.append(LDMessage(role=msg.role, content=content))
+        Legacy messages (assistant/user messages containing ``{{message_history}}``
+        or ``{{response_to_evaluate}}``) are stripped from the config; the runner
+        was already created from the judge AI config (which carries the system
+        message), so only the plain-text evaluation payload is needed here.
 
-        return messages
-
-    def _interpolate_message(self, content: str, variables: Dict[str, str]) -> str:
-        """
-        Interpolates message content with variables using Mustache templating.
-
-        :param content: The message content template
-        :param variables: Variables to interpolate
-        :return: Interpolated message content
+        :param input_text: The input text (message history)
+        :param output_text: The output text to evaluate
+        :return: Formatted evaluation input string
         """
-        # Use chevron (Mustache) for templating, with no escaping
-        return chevron.render(content, variables)
+        _strip_legacy_judge_messages(self._ai_config.messages or [])
+        return f"MESSAGE HISTORY:\n{input_text}\n\nRESPONSE TO EVALUATE:\n{output_text}"
 
     def _parse_evaluation_response(self, data: Dict[str, Any]) -> Optional[Tuple[float, str]]:
         """

packages/sdk/server-ai/src/ldai/providers/runner.py

@@ -16,13 +16,13 @@ class Runner(Protocol):
 
     async def run(
         self,
-        input: Any,
+        input: str,
         output_type: Optional[Dict[str, Any]] = None,
     ) -> RunnerResult:
         """
-        Execute the runner with the given input.
+        Execute the runner with the given input string.
 
-        :param input: The input to the runner.
+        :param input: The string input to the runner.
         :param output_type: Optional JSON schema for structured output.
         :return: RunnerResult containing content, metrics, raw, and parsed fields.
         """