fix: Make judge runners non-multi-turn

Add a multiTurn parameter (default true) on provider model runners and
RunnerFactory.createModel. When false, the runner does not persist the
user prompt and assistant reply back into its conversation history, so
each run() call starts fresh from the seeded config messages.

Judges now construct their underlying runner with multiTurn=false so
successive evaluate() calls on a shared Judge instance do not see each
other's prompts and responses. Without this, every evaluation after the
first contaminated the judge model's input with prior conversations and
concurrent evaluations raced on the mutable history.

Also fix Judge.evaluateMessages to render messages as "<role>: <content>"
joined by newlines, preserving speaker identity in the message history
section the judge model receives.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: jsonbailey

packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts

@@ -180,4 +180,68 @@ describe('LangChainModelRunner', () => {
       expect(secondCallMessages[3].content).toBe('Q2');
     });
   });
+
+  describe('multiTurn=false (stateless)', () => {
+    const configWithMessages: LDAICompletionConfig = {
+      ...baseConfig,
+      messages: [{ role: 'system', content: 'You are a judge.' }],
+    };
+
+    it('does not accumulate history across successful calls', async () => {
+      const statelessRunner = new LangChainModelRunner(
+        mockLLM,
+        configWithMessages,
+        mockLogger,
+        false,
+      );
+
+      mockLLM.invoke
+        .mockResolvedValueOnce(new AIMessage('First response'))
+        .mockResolvedValueOnce(new AIMessage('Second response'));
+
+      await statelessRunner.run('First question');
+      await statelessRunner.run('Second question');
+
+      const firstCallMessages = mockLLM.invoke.mock.calls[0][0];
+      const secondCallMessages = mockLLM.invoke.mock.calls[1][0];
+      expect(firstCallMessages).toHaveLength(2);
+      expect(firstCallMessages[0].content).toBe('You are a judge.');
+      expect(firstCallMessages[1].content).toBe('First question');
+      expect(secondCallMessages).toHaveLength(2);
+      expect(secondCallMessages[0].content).toBe('You are a judge.');
+      expect(secondCallMessages[1].content).toBe('Second question');
+    });
+
+    it('keeps the internal chat history length pinned to the seeded config messages', async () => {
+      const statelessRunner = new LangChainModelRunner(
+        mockLLM,
+        configWithMessages,
+        mockLogger,
+        false,
+      );
+
+      mockLLM.invoke.mockResolvedValue(new AIMessage('response'));
+
+      await statelessRunner.run('Q1');
+      await statelessRunner.run('Q2');
+
+      // eslint-disable-next-line no-underscore-dangle
+      const messages = await (statelessRunner as any)._chatHistory.getMessages();
+      expect(messages).toHaveLength(1);
+    });
+
+    it('defaults to multiTurn=true when the parameter is omitted', async () => {
+      const defaultRunner = new LangChainModelRunner(mockLLM, configWithMessages, mockLogger);
+
+      mockLLM.invoke
+        .mockResolvedValueOnce(new AIMessage('Answer 1'))
+        .mockResolvedValueOnce(new AIMessage('Answer 2'));
+
+      await defaultRunner.run('Q1');
+      await defaultRunner.run('Q2');
+
+      const secondCallMessages = mockLLM.invoke.mock.calls[1][0];
+      expect(secondCallMessages).toHaveLength(4);
+    });
+  });
 });

packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts

@@ -20,26 +20,37 @@ import { convertMessagesToLangChain, getAIMetricsFromResponse } from './LangChai
 export class LangChainModelRunner implements Runner {
   private _llm: BaseChatModel;
   private _chatHistory: InMemoryChatMessageHistory;
+  private _multiTurn: boolean;
   private _logger?: LDLogger;
 
-  constructor(llm: BaseChatModel, config: LDAICompletionConfig, logger?: LDLogger) {
+  constructor(
+    llm: BaseChatModel,
+    config: LDAICompletionConfig,
+    logger?: LDLogger,
+    multiTurn: boolean = true,
+  ) {
     this._llm = llm;
     this._chatHistory = new InMemoryChatMessageHistory(
       convertMessagesToLangChain(config.messages ?? []),
     );
+    this._multiTurn = multiTurn;
     this._logger = logger;
   }
 
   /**
    * Run the LangChain model with the given user prompt.
    *
    * The runner maintains a LangChain `InMemoryChatMessageHistory` that is
-   * initialized from any messages on the AI config (system prompt, etc.) and
-   * grows with each successful call. On every invocation the user prompt is
-   * appended to the existing history before being sent to the model. When the
-   * call succeeds and produces non-empty content, the user prompt and the
-   * assistant's reply are persisted to the history; failed calls leave the
-   * history unchanged so the next call can retry cleanly.
+   * initialized from any messages on the AI config (system prompt, etc.). On
+   * every invocation the user prompt is appended to the existing history
+   * before being sent to the model. When `multiTurn` is `true` (the default)
+   * and the call succeeds with non-empty content, the user prompt and the
+   * assistant's reply are persisted to the history so subsequent calls
+   * continue the conversation. When `multiTurn` is `false`, history is
+   * treated as read-only — useful for stateless runners (e.g. judges) where
+   * every call should see only the initial config messages plus the current
+   * input. Failed calls leave the history unchanged so the next call can
+   * retry cleanly.
    *
    * @param input The user prompt string.
    * @param outputType Optional JSON schema for structured output. When provided,
@@ -56,7 +67,7 @@ export class LangChainModelRunner implements Runner {
         ? await this._runStructured(langchainMessages, outputType)
         : await this._runCompletion(langchainMessages);
 
-    if (result.metrics.success && result.content) {
+    if (result.metrics.success && result.content && this._multiTurn) {
       await this._chatHistory.addUserMessage(input);
       await this._chatHistory.addAIMessage(result.content);
     }

packages/ai-providers/server-ai-langchain/src/LangChainRunnerFactory.ts

@@ -26,10 +26,18 @@ export class LangChainRunnerFactory extends AIProvider {
 
   /**
    * Create a model runner from a completion AI configuration.
+   *
+   * @param config The completion (or judge) AI configuration.
+   * @param multiTurn Whether the runner should accumulate conversation history
+   *   across successive `run()` calls. Defaults to `true` (chat semantics).
+   *   Pass `false` for stateless runners such as judges.
    */
-  async createModel(config: LDAICompletionConfig): Promise<LangChainModelRunner> {
+  async createModel(
+    config: LDAICompletionConfig,
+    multiTurn: boolean = true,
+  ): Promise<LangChainModelRunner> {
     const llm = await createLangChainModel(config);
-    return new LangChainModelRunner(llm, config, this._logger);
+    return new LangChainModelRunner(llm, config, this._logger, multiTurn);
   }
 
   /**

packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts

@@ -231,4 +231,89 @@ describe('OpenAIModelRunner', () => {
       ]);
     });
   });
+
+  describe('multiTurn=false (stateless)', () => {
+    const configWithMessages: LDAICompletionConfig = {
+      ...baseConfig,
+      messages: [{ role: 'system', content: 'You are a judge.' }],
+    };
+
+    it('does not accumulate history across successful calls', async () => {
+      const statelessRunner = new OpenAIModelRunner(
+        mockOpenAI,
+        configWithMessages,
+        undefined,
+        false,
+      );
+
+      (mockOpenAI.chat.completions.create as jest.Mock)
+        .mockResolvedValueOnce({
+          choices: [{ message: { content: 'First response' } }],
+          usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+        } as any)
+        .mockResolvedValueOnce({
+          choices: [{ message: { content: 'Second response' } }],
+          usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+        } as any);
+
+      await statelessRunner.run('First question');
+      await statelessRunner.run('Second question');
+
+      const firstCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[0][0];
+      const secondCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[1][0];
+      expect(firstCallArgs.messages).toEqual([
+        { role: 'system', content: 'You are a judge.' },
+        { role: 'user', content: 'First question' },
+      ]);
+      expect(secondCallArgs.messages).toEqual([
+        { role: 'system', content: 'You are a judge.' },
+        { role: 'user', content: 'Second question' },
+      ]);
+    });
+
+    it('keeps the internal history length pinned to the seeded config messages', async () => {
+      const statelessRunner = new OpenAIModelRunner(
+        mockOpenAI,
+        configWithMessages,
+        undefined,
+        false,
+      );
+
+      (mockOpenAI.chat.completions.create as jest.Mock).mockResolvedValue({
+        choices: [{ message: { content: 'response' } }],
+        usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+      } as any);
+
+      await statelessRunner.run('Q1');
+      await statelessRunner.run('Q2');
+
+      // eslint-disable-next-line no-underscore-dangle
+      expect((statelessRunner as any)._history).toHaveLength(1);
+    });
+
+    it('defaults to multiTurn=true when the parameter is omitted', async () => {
+      const defaultRunner = new OpenAIModelRunner(mockOpenAI, configWithMessages);
+
+      (mockOpenAI.chat.completions.create as jest.Mock)
+        .mockResolvedValueOnce({
+          choices: [{ message: { content: 'Answer 1' } }],
+          usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+        } as any)
+        .mockResolvedValueOnce({
+          choices: [{ message: { content: 'Answer 2' } }],
+          usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+        } as any);
+
+      await defaultRunner.run('Q1');
+      await defaultRunner.run('Q2');
+
+      const secondCallArgs = (mockOpenAI.chat.completions.create as jest.Mock).mock.calls[1][0];
+      expect(secondCallArgs.messages).toEqual([
+        { role: 'system', content: 'You are a judge.' },
+        { role: 'user', content: 'Q1' },
+        { role: 'assistant', content: 'Answer 1' },
+        { role: 'user', content: 'Q2' },
+      ]);
+    });
+  });
 });

packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts

@@ -21,26 +21,37 @@ export class OpenAIModelRunner implements Runner {
   private _modelName: string;
   private _parameters: Record<string, unknown>;
   private _history: LDMessage[];
+  private _multiTurn: boolean;
   private _logger?: LDLogger;
 
-  constructor(client: OpenAI, config: LDAICompletionConfig, logger?: LDLogger) {
+  constructor(
+    client: OpenAI,
+    config: LDAICompletionConfig,
+    logger?: LDLogger,
+    multiTurn: boolean = true,
+  ) {
     this._client = client;
     this._modelName = config.model?.name ?? '';
     this._parameters = { ...(config.model?.parameters ?? {}) };
     this._history = [...(config.messages ?? [])];
+    this._multiTurn = multiTurn;
     this._logger = logger;
   }
 
   /**
    * Run the OpenAI model with the given user prompt.
    *
    * The runner maintains a conversation history that is initialized from any
-   * messages on the AI config (system prompt, instructions, etc.) and grows
-   * with each successful call. On every invocation the user prompt is appended
-   * to the existing history before being sent to the model. When the call
-   * succeeds and produces non-empty content, the user prompt and the
-   * assistant's reply are persisted to the history; failed calls leave the
-   * history unchanged so the next call can retry cleanly.
+   * messages on the AI config (system prompt, instructions, etc.). On every
+   * invocation the user prompt is appended to the existing history before
+   * being sent to the model. When `multiTurn` is `true` (the default) and the
+   * call succeeds with non-empty content, the user prompt and the assistant's
+   * reply are persisted to the history so subsequent calls continue the
+   * conversation. When `multiTurn` is `false`, history is treated as
+   * read-only — useful for stateless runners (e.g. judges) where every call
+   * should see only the initial config messages plus the current input.
+   * Failed calls leave the history unchanged so the next call can retry
+   * cleanly.
    *
    * @param input The user prompt string.
    * @param outputType Optional JSON schema for structured output. When provided,
@@ -55,7 +66,7 @@ export class OpenAIModelRunner implements Runner {
         ? await this._runStructured(messages, outputType)
         : await this._runCompletion(messages);
 
-    if (result.metrics.success && result.content) {
+    if (result.metrics.success && result.content && this._multiTurn) {
       this._history.push(userMessage);
       this._history.push({ role: 'assistant', content: result.content });
     }

packages/ai-providers/server-ai-openai/src/OpenAIRunnerFactory.ts

@@ -28,9 +28,17 @@ export class OpenAIRunnerFactory extends AIProvider {
 
   /**
    * Create a model runner from a completion AI configuration.
+   *
+   * @param config The completion (or judge) AI configuration.
+   * @param multiTurn Whether the runner should accumulate conversation history
+   *   across successive `run()` calls. Defaults to `true` (chat semantics).
+   *   Pass `false` for stateless runners such as judges.
    */
-  async createModel(config: LDAICompletionConfig): Promise<OpenAIModelRunner> {
-    return new OpenAIModelRunner(this._client, config, this._logger);
+  async createModel(
+    config: LDAICompletionConfig,
+    multiTurn: boolean = true,
+  ): Promise<OpenAIModelRunner> {
+    return new OpenAIModelRunner(this._client, config, this._logger, multiTurn);
   }
 
   /**