fix!: build judge input as string and narrow Runner.run signature

Aligns the JS AI SDK with the spec change implemented in
launchdarkly/python-server-sdk-ai#165 and launchdarkly/sdk-specs#160.

Judges now build a single formatted string ("MESSAGE HISTORY:\n...\n\n
RESPONSE TO EVALUATE:\n...") and pass it to the runner instead of an
interpolated message list. Legacy judge configs that contain
{{message_history}} or {{response_to_evaluate}} placeholders in
non-system messages are stripped at config-construction time so old and
new flag values both work without behavioral surprises.

BREAKING CHANGE: Runner.run is narrowed from
`run(input: string | LDMessage[], outputType?)` to
`run(input: string, outputType?)`. The OpenAI, LangChain, and Vercel
provider runners no longer accept a pre-built message array; they
always prepend any config messages and append the prompt as a user
turn. The Judge no longer interpolates {{message_history}} or
{{response_to_evaluate}} into config messages — the SDK builds the
input string directly and the runner receives that string verbatim.
The "Judge configuration must include messages" early-return was
removed; a judge with no messages now proceeds to invoke the runner
with the formatted input.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: jsonbailey

packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts

@@ -29,19 +29,20 @@ export class LangChainModelRunner implements Runner {
   }
 
   /**
-   * Run the LangChain model with the given prompt.
+   * Run the LangChain model with the given user prompt.
    *
-   * @param input The user prompt string or a pre-built message array to send to the model.
-   *   When a string is provided, config messages are prepended before the user prompt.
-   *   When an {@link LDMessage} array is provided, it is used as-is (config messages are
-   *   not prepended).
+   * Prepends any messages defined in the AI config (system prompt, etc.) before
+   * the user prompt.
+   *
+   * @param input The user prompt string.
    * @param outputType Optional JSON schema for structured output. When provided,
    *   the parsed result is exposed via {@link RunnerResult.parsed}.
    */
-  async run(input: string | LDMessage[], outputType?: Record<string, unknown>): Promise<RunnerResult> {
-    const messages: LDMessage[] = Array.isArray(input)
-      ? input
-      : [...(this._config.messages ?? []), { role: 'user', content: input }];
+  async run(input: string, outputType?: Record<string, unknown>): Promise<RunnerResult> {
+    const messages: LDMessage[] = [
+      ...(this._config.messages ?? []),
+      { role: 'user', content: input },
+    ];
 
     if (outputType !== undefined) {
       return this._runStructured(messages, outputType);

packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts

@@ -75,27 +75,6 @@ describe('OpenAIModelRunner', () => {
       });
     });
 
-    it('passes a LDMessage[] input directly without prepending config messages', async () => {
-      const mockResponse = {
-        choices: [{ message: { content: 'Evaluation result' } }],
-        usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 },
-      };
-      (mockOpenAI.chat.completions.create as jest.Mock).mockResolvedValue(mockResponse as any);
-
-      const messages = [
-        { role: 'system' as const, content: 'You are a judge' },
-        { role: 'user' as const, content: 'Rate this: hello' },
-      ];
-      const result = await runner.run(messages);
-
-      expect(mockOpenAI.chat.completions.create).toHaveBeenCalledWith({
-        model: 'gpt-3.5-turbo',
-        messages,
-      });
-      expect(result.content).toBe('Evaluation result');
-      expect(result.metrics.success).toBe(true);
-    });
-
     it('marks the result unsuccessful when response has no content', async () => {
       const mockResponse = { choices: [{ message: {} }] };
       (mockOpenAI.chat.completions.create as jest.Mock).mockResolvedValue(mockResponse as any);

packages/ai-providers/server-ai-openai/src/OpenAIAgentRunner.ts

@@ -48,7 +48,7 @@ export class OpenAIAgentRunner implements Runner {
 
   async run(input: string, _outputType?: Record<string, unknown>): Promise<RunnerResult> {
     try {
-      const result = await this._agentRun(this._agent, String(input), { maxTurns: MAX_TURNS });
+      const result = await this._agentRun(this._agent, input, { maxTurns: MAX_TURNS });
 
       const toolCalls = getToolCallsFromRunItems(result.newItems ?? []).reduce(
         (acc: string[], fnName: string) => {

packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts

@@ -32,20 +32,20 @@ export class OpenAIModelRunner implements Runner {
   }
 
   /**
-   * Run the OpenAI model with the given prompt or message array.
+   * Run the OpenAI model with the given user prompt.
    *
-   * When `input` is a string it is wrapped as a user turn and appended to any
-   * messages defined in the config. When `input` is already a `LDMessage[]`
-   * (e.g. when called from the Judge evaluation path) it is used as-is.
+   * Prepends any messages defined in the AI config (system prompt,
+   * instructions, etc.) before the user prompt.
    *
-   * @param input The user prompt string, or a pre-built message array.
+   * @param input The user prompt string.
    * @param outputType Optional JSON schema for structured output. When provided,
    *   the response is parsed and exposed via {@link RunnerResult.parsed}.
    */
-  async run(input: string | LDMessage[], outputType?: Record<string, unknown>): Promise<RunnerResult> {
-    const messages: LDMessage[] = Array.isArray(input)
-      ? input
-      : [...(this._config.messages ?? []), { role: 'user', content: input }];
+  async run(input: string, outputType?: Record<string, unknown>): Promise<RunnerResult> {
+    const messages: LDMessage[] = [
+      ...(this._config.messages ?? []),
+      { role: 'user', content: input },
+    ];
 
     if (outputType !== undefined) {
       return this._runStructured(messages, outputType);

packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts

@@ -36,19 +36,20 @@ export class VercelModelRunner implements Runner {
   }
 
   /**
-   * Run the Vercel AI model with the given prompt.
+   * Run the Vercel AI model with the given user prompt.
    *
-   * @param input The user prompt string, or a pre-built message array. When a
-   *   string is supplied the config's system messages are prepended automatically.
-   *   When a `LDMessage[]` is supplied it is used as-is (config messages are NOT
-   *   prepended — the caller is responsible for the full message list).
+   * Prepends any messages defined in the AI config (system prompt, etc.) before
+   * the user prompt.
+   *
+   * @param input The user prompt string.
    * @param outputType Optional JSON schema for structured output. When provided,
    *   the parsed object is exposed via {@link RunnerResult.parsed}.
    */
-  async run(input: string | LDMessage[], outputType?: Record<string, unknown>): Promise<RunnerResult> {
-    const messages: LDMessage[] = Array.isArray(input)
-      ? input
-      : [...(this._config.messages ?? []), { role: 'user', content: input }];
+  async run(input: string, outputType?: Record<string, unknown>): Promise<RunnerResult> {
+    const messages: LDMessage[] = [
+      ...(this._config.messages ?? []),
+      { role: 'user', content: input },
+    ];
 
     if (outputType !== undefined) {
       return this._runStructured(messages, outputType);

packages/sdk/server-ai/__tests__/Judge.test.ts

@@ -2,10 +2,74 @@ import { LDLogger } from '@launchdarkly/js-server-sdk-common';
 
 import { LDAIConfigTracker } from '../src/api/config/LDAIConfigTracker';
 import { LDAIJudgeConfig, LDMessage } from '../src/api/config/types';
-import { Judge } from '../src/api/judge/Judge';
+import { Judge, stripLegacyJudgeMessages } from '../src/api/judge/Judge';
 import { RunnerResult } from '../src/api/model/types';
 import { Runner } from '../src/api/providers/Runner';
 
+describe('stripLegacyJudgeMessages', () => {
+  it('strips assistant messages containing {{message_history}}', () => {
+    const messages: LDMessage[] = [
+      { role: 'system', content: 'You are a judge.' },
+      { role: 'assistant', content: 'Here is the history: {{message_history}}' },
+    ];
+    const result = stripLegacyJudgeMessages(messages);
+    expect(result).toHaveLength(1);
+    expect(result[0].role).toBe('system');
+  });
+
+  it('strips user messages containing {{response_to_evaluate}}', () => {
+    const messages: LDMessage[] = [
+      { role: 'system', content: 'You are a judge.' },
+      { role: 'user', content: 'Evaluate: {{response_to_evaluate}}' },
+    ];
+    const result = stripLegacyJudgeMessages(messages);
+    expect(result).toHaveLength(1);
+    expect(result[0].role).toBe('system');
+  });
+
+  it('strips all legacy template messages from a typical legacy config', () => {
+    const messages: LDMessage[] = [
+      { role: 'system', content: 'You are a judge.' },
+      { role: 'assistant', content: '{{message_history}}' },
+      { role: 'user', content: '{{response_to_evaluate}}' },
+    ];
+    const result = stripLegacyJudgeMessages(messages);
+    expect(result).toHaveLength(1);
+    expect(result[0].role).toBe('system');
+  });
+
+  it('does not strip system messages even when they contain template variables', () => {
+    const messages: LDMessage[] = [
+      {
+        role: 'system',
+        content: 'Judge using {{message_history}} and {{response_to_evaluate}}.',
+      },
+    ];
+    const result = stripLegacyJudgeMessages(messages);
+    expect(result).toHaveLength(1);
+    expect(result[0].role).toBe('system');
+  });
+
+  it('leaves non-system messages without template variables untouched', () => {
+    const messages: LDMessage[] = [
+      { role: 'system', content: 'You are a judge.' },
+      { role: 'user', content: 'This is a regular message.' },
+    ];
+    const result = stripLegacyJudgeMessages(messages);
+    expect(result).toHaveLength(2);
+  });
+
+  it('returns an empty array for an empty input', () => {
+    expect(stripLegacyJudgeMessages([])).toEqual([]);
+  });
+
+  it('passes a new-style system-only config through unchanged', () => {
+    const messages: LDMessage[] = [{ role: 'system', content: 'You are a judge.' }];
+    const result = stripLegacyJudgeMessages(messages);
+    expect(result).toEqual(messages);
+  });
+});
+
 describe('Judge', () => {
   let mockRunner: jest.Mocked<Runner>;
   let mockTracker: jest.Mocked<LDAIConfigTracker>;
@@ -37,14 +101,7 @@ describe('Judge', () => {
     judgeConfig = {
       key: 'test-judge',
       enabled: true,
-      messages: [
-        { role: 'system', content: 'You are a helpful judge that evaluates AI responses.' },
-        {
-          role: 'user',
-          content:
-            'Evaluate and report scores for important metrics: Input: {{message_history}}, Output: {{response_to_evaluate}}',
-        },
-      ],
+      messages: [{ role: 'system', content: 'You are a helpful judge that evaluates AI responses.' }],
       model: { name: 'gpt-4' },
       provider: { name: 'openai' },
       createTracker: () => mockTracker,
@@ -161,21 +218,33 @@ describe('Judge', () => {
       });
 
       expect(mockRunner.run).toHaveBeenCalledWith(
-        expect.arrayContaining([
-          expect.objectContaining({
-            role: 'system',
-            content: 'You are a helpful judge that evaluates AI responses.',
-          }),
-          expect.objectContaining({
-            role: 'user',
-            content:
-              'Evaluate and report scores for important metrics: Input: What is the capital of France?, Output: Paris is the capital of France.',
-          }),
-        ]),
+        'MESSAGE HISTORY:\nWhat is the capital of France?\n\nRESPONSE TO EVALUATE:\nParis is the capital of France.',
         expect.any(Object), // evaluation schema
       );
     });
 
+    it('passes a string input to the runner (not a message list)', async () => {
+      const mockRunnerResult: RunnerResult = {
+        content: '',
+        parsed: {
+          score: 0.85,
+          reasoning: 'Good answer.',
+        },
+        metrics: { success: true },
+      };
+
+      mockTracker.trackMetricsOf.mockImplementation(async (extractor, func) => func());
+      mockRunner.run.mockResolvedValue(mockRunnerResult);
+
+      await judge.evaluate('What is AI?', 'AI is artificial intelligence.');
+
+      expect(mockRunner.run).toHaveBeenCalledTimes(1);
+      const inputArg = mockRunner.run.mock.calls[0][0];
+      expect(typeof inputArg).toBe('string');
+      expect(inputArg).toContain('MESSAGE HISTORY:\nWhat is AI?');
+      expect(inputArg).toContain('RESPONSE TO EVALUATE:\nAI is artificial intelligence.');
+    });
+
     it('returns evaluation result with correct evaluationMetricKey for tracker integration', async () => {
       const mockRunnerResult: RunnerResult = {
         content: '',
@@ -412,25 +481,29 @@ describe('Judge', () => {
       });
     });
 
-    it('returns error result when messages are missing', async () => {
+    it('proceeds (does not error early) when messages is undefined', async () => {
       const configWithoutMessages: LDAIJudgeConfig = {
         ...judgeConfig,
         messages: undefined,
       };
       const judgeWithoutMessages = new Judge(configWithoutMessages, mockRunner, 1.0, mockLogger);
 
+      const mockRunnerResult: RunnerResult = {
+        content: '',
+        parsed: {
+          score: 0.7,
+          reasoning: 'Acceptable response.',
+        },
+        metrics: { success: true },
+      };
+      mockTracker.trackMetricsOf.mockImplementation(async (extractor, func) => func());
+      mockRunner.run.mockResolvedValue(mockRunnerResult);
+
       const result = await judgeWithoutMessages.evaluate('test input', 'test output');
 
-      expect(result).toEqual({
-        success: false,
-        sampled: true,
-        errorMessage: 'Judge configuration must include messages',
-        judgeConfigKey: 'test-judge',
-      });
-      expect(mockLogger.warn).toHaveBeenCalledWith(
-        'Judge configuration must include messages',
-        mockTrackData,
-      );
+      expect(result.sampled).toBe(true);
+      expect(result.success).toBe(true);
+      expect(mockRunner.run).toHaveBeenCalledTimes(1);
     });
 
     it('returns result with success false when parsed is undefined or has no score/reasoning', async () => {
@@ -588,17 +661,7 @@ describe('Judge', () => {
       });
 
       expect(mockRunner.run).toHaveBeenCalledWith(
-        expect.arrayContaining([
-          expect.objectContaining({
-            role: 'system',
-            content: 'You are a helpful judge that evaluates AI responses.',
-          }),
-          expect.objectContaining({
-            role: 'user',
-            content:
-              'Evaluate and report scores for important metrics: Input: What is the capital of France?\r\nParis is the capital of France., Output: Paris is the capital of France.',
-          }),
-        ]),
+        'MESSAGE HISTORY:\nWhat is the capital of France?\r\nParis is the capital of France.\n\nRESPONSE TO EVALUATE:\nParis is the capital of France.',
         expect.any(Object), // evaluation schema
       );
     });
@@ -626,28 +689,19 @@ describe('Judge', () => {
     });
   });
 
-  describe('_constructEvaluationMessages', () => {
+  describe('_buildEvaluationInput', () => {
     let judge: Judge;
 
     beforeEach(() => {
       judge = new Judge(judgeConfig, mockRunner, 1.0, mockLogger);
     });
 
-    it('constructs evaluation messages correctly', () => {
+    it('builds the evaluation string in the expected format', () => {
       // eslint-disable-next-line no-underscore-dangle
-      const constructMessages = (judge as any)._constructEvaluationMessages.bind(judge);
-      const messages = constructMessages('test input', 'test output');
+      const buildInput = (judge as any)._buildEvaluationInput.bind(judge);
+      const input = buildInput('hello', 'world');
 
-      expect(messages).toHaveLength(2);
-      expect(messages[0]).toEqual({
-        role: 'system',
-        content: 'You are a helpful judge that evaluates AI responses.',
-      });
-      expect(messages[1]).toEqual({
-        role: 'user',
-        content:
-          'Evaluate and report scores for important metrics: Input: test input, Output: test output',
-      });
+      expect(input).toBe('MESSAGE HISTORY:\nhello\n\nRESPONSE TO EVALUATE:\nworld');
     });
   });