feat(server-sdk-ai): add Evaluator class for judge orchestration (AIC-1657)

Introduces `Evaluator` wrapping judges and JudgeConfiguration. The evaluator
runs all configured judges in parallel, warns+skips on missing judge keys, and
intentionally does NOT call tracker.trackJudgeResult — that responsibility
belongs in the managed layer. Attaches Evaluator to LDAICompletionConfig and
LDAIAgentConfig via createChat/createAgent. Adds Evaluator.noop() static factory.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: jsonbailey

packages/sdk/server-ai/__tests__/Evaluator.test.ts

@@ -0,0 +1,164 @@
+import { LDAIJudgeConfig } from '../src/api/config/types';
+import { Evaluator } from '../src/api/judge/Evaluator';
+import { Judge } from '../src/api/judge/Judge';
+import { LDJudgeResult } from '../src/api/judge/types';
+import { AIProvider } from '../src/api/providers/AIProvider';
+
+function makeJudgeConfig(key: string): LDAIJudgeConfig {
+  return {
+    key,
+    enabled: true,
+    evaluationMetricKey: '$ld:ai:judge:quality',
+    messages: [{ role: 'system', content: 'You are a judge.' }],
+    createTracker: () => ({}) as any,
+  };
+}
+
+function makeProvider(): jest.Mocked<AIProvider> {
+  return {
+    invokeModel: jest.fn(),
+    invokeStructuredModel: jest.fn(),
+  } as any;
+}
+
+describe('Evaluator', () => {
+  describe('noop()', () => {
+    it('returns an empty result array', async () => {
+      const evaluator = Evaluator.noop();
+      const results = await evaluator.evaluate('input', 'output');
+      expect(results).toEqual([]);
+    });
+
+    it('has empty judges map', () => {
+      const evaluator = Evaluator.noop();
+      expect(evaluator.judges.size).toBe(0);
+    });
+
+    it('has empty judge configuration', () => {
+      const evaluator = Evaluator.noop();
+      expect(evaluator.judgeConfiguration.judges).toEqual([]);
+    });
+  });
+
+  describe('evaluate()', () => {
+    it('calls each configured judge and returns results', async () => {
+      const mockProvider = makeProvider();
+      const judgeConfig = makeJudgeConfig('judge-1');
+
+      const mockResult: LDJudgeResult = {
+        success: true,
+        sampled: true,
+        score: 0.9,
+        reasoning: 'Good response',
+        metricKey: '$ld:ai:judge:quality',
+        judgeConfigKey: 'judge-1',
+      };
+
+      const judge = new Judge(judgeConfig, mockProvider);
+      jest.spyOn(judge, 'evaluate').mockResolvedValue(mockResult);
+
+      const judges = new Map([['judge-1', judge]]);
+      const evaluator = new Evaluator(judges, { judges: [{ key: 'judge-1', samplingRate: 1.0 }] });
+
+      const results = await evaluator.evaluate('user input', 'ai output');
+
+      expect(results).toHaveLength(1);
+      expect(results[0]).toEqual(mockResult);
+      expect(judge.evaluate).toHaveBeenCalledWith('user input', 'ai output', 1.0);
+    });
+
+    it('warns and skips when judge key is not found in judges map', async () => {
+      const mockLogger = { warn: jest.fn(), debug: jest.fn(), info: jest.fn(), error: jest.fn() };
+      const judges = new Map<string, Judge>();
+      const evaluator = new Evaluator(
+        judges,
+        { judges: [{ key: 'missing-judge', samplingRate: 1.0 }] },
+        mockLogger,
+      );
+
+      const results = await evaluator.evaluate('input', 'output');
+
+      expect(mockLogger.warn).toHaveBeenCalledWith(expect.stringContaining('missing-judge'));
+      // Missing judge is skipped (not an error result), so results array is empty
+      expect(results).toEqual([]);
+    });
+
+    it('returns error result when judge throws', async () => {
+      const mockProvider = makeProvider();
+      const judgeConfig = makeJudgeConfig('judge-err');
+
+      const judge = new Judge(judgeConfig, mockProvider);
+      jest.spyOn(judge, 'evaluate').mockRejectedValue(new Error('evaluation error'));
+
+      const judges = new Map([['judge-err', judge]]);
+      const evaluator = new Evaluator(judges, {
+        judges: [{ key: 'judge-err', samplingRate: 1.0 }],
+      });
+
+      const results = await evaluator.evaluate('input', 'output');
+
+      expect(results).toHaveLength(1);
+      expect(results[0].success).toBe(false);
+      expect(results[0].sampled).toBe(true);
+      expect(results[0].errorMessage).toBe('evaluation error');
+    });
+
+    it('does NOT call tracker.trackJudgeResult', async () => {
+      const mockProvider = makeProvider();
+      const judgeConfig = makeJudgeConfig('judge-1');
+
+      const mockResult: LDJudgeResult = {
+        success: true,
+        sampled: true,
+        score: 0.8,
+        reasoning: 'ok',
+        metricKey: '$ld:ai:judge:quality',
+      };
+
+      const judge = new Judge(judgeConfig, mockProvider);
+      jest.spyOn(judge, 'evaluate').mockResolvedValue(mockResult);
+
+      const judges = new Map([['judge-1', judge]]);
+      const evaluator = new Evaluator(judges, { judges: [{ key: 'judge-1', samplingRate: 1.0 }] });
+
+      // No tracker — if Evaluator tried to call trackJudgeResult this would throw or fail
+      await evaluator.evaluate('input', 'output');
+
+      // Test passes if no error is thrown (no tracker involved)
+      expect(true).toBe(true);
+    });
+
+    it('runs multiple judges in parallel and returns all results', async () => {
+      const makeJudge = (key: string, score: number): Judge => {
+        const mockProvider = makeProvider();
+        const jc = makeJudgeConfig(key);
+        const j = new Judge(jc, mockProvider);
+        jest.spyOn(j, 'evaluate').mockResolvedValue({
+          success: true,
+          sampled: true,
+          score,
+          reasoning: 'ok',
+          metricKey: '$ld:ai:judge:quality',
+        });
+        return j;
+      };
+
+      const judges = new Map([
+        ['judge-a', makeJudge('judge-a', 0.5)],
+        ['judge-b', makeJudge('judge-b', 0.9)],
+      ]);
+      const evaluator = new Evaluator(judges, {
+        judges: [
+          { key: 'judge-a', samplingRate: 1.0 },
+          { key: 'judge-b', samplingRate: 1.0 },
+        ],
+      });
+
+      const results = await evaluator.evaluate('input', 'output');
+
+      expect(results).toHaveLength(2);
+      const scores = results.map((r) => r.score).sort();
+      expect(scores).toEqual([0.5, 0.9]);
+    });
+  });
+});

packages/sdk/server-ai/src/LDAIClientImpl.ts

@@ -22,6 +22,7 @@ import {
 } from './api/config';
 import { LDAIConfigFlagValue, LDAIConfigUtils } from './api/config/LDAIConfigUtils';
 import { AgentGraphDefinition, LDAgentGraphFlagValue, LDGraphTracker } from './api/graph';
+import { Evaluator } from './api/judge/Evaluator';
 import { Judge } from './api/judge/Judge';
 import { LDAIClient } from './api/LDAIClient';
 import { AIProviderFactory, SupportedAIProvider } from './api/providers';
@@ -170,6 +171,26 @@ export class LDAIClientImpl implements LDAIClient {
     return judges;
   }
 
+  private async _buildEvaluator(
+    judgeConfigs: LDJudge[],
+    context: LDContext,
+    variables?: Record<string, unknown>,
+    defaultAiProvider?: SupportedAIProvider,
+  ): Promise<Evaluator> {
+    if (judgeConfigs.length === 0) {
+      return Evaluator.noop();
+    }
+
+    const judgesRecord = await this._initializeJudges(
+      judgeConfigs,
+      context,
+      variables,
+      defaultAiProvider,
+    );
+    const judgesMap = new Map<string, Judge>(Object.entries(judgesRecord));
+    return new Evaluator(judgesMap, { judges: judgeConfigs }, this._logger);
+  }
+
   private async _completionConfig(
     key: string,
     context: LDContext,
@@ -318,14 +339,23 @@ export class LDAIClientImpl implements LDAIClient {
       return undefined;
     }
 
-    const judges = await this._initializeJudges(
+    const evaluator = await this._buildEvaluator(
       config.judgeConfiguration?.judges ?? [],
       context,
       variables,
       defaultAiProvider,
     );
 
-    return new TrackedChat(config, provider, judges, this._logger);
+    // Attach the evaluator to the config for use by the managed layer
+    const configWithEvaluator: LDAICompletionConfig = { ...config, evaluator };
+
+    // Build the legacy judges record for TrackedChat backward compat
+    const judges: Record<string, Judge> = {};
+    evaluator.judges.forEach((judge, k) => {
+      judges[k] = judge;
+    });
+
+    return new TrackedChat(configWithEvaluator, provider, judges, this._logger);
   }
 
   async createJudge(

packages/sdk/server-ai/src/api/config/types.ts

@@ -1,3 +1,4 @@
+import type { Evaluator } from '../judge/Evaluator';
 import { LDAIConfigTracker } from './LDAIConfigTracker';
 
 // ============================================================================
@@ -220,6 +221,11 @@ export interface LDAIAgentConfig extends LDAIConfig {
    * Root-level tools map keyed by tool name. Distinct from model.parameters.tools[].
    */
   tools?: { [toolName: string]: LDTool };
+  /**
+   * Evaluator for this agent config. Populated by createAgent.
+   * Internal; not part of the flag value shape.
+   */
+  evaluator?: Evaluator;
 }
 
 /**
@@ -239,6 +245,11 @@ export interface LDAICompletionConfig extends LDAIConfig {
    * Root-level tools map keyed by tool name. Distinct from model.parameters.tools[].
    */
   tools?: { [toolName: string]: LDTool };
+  /**
+   * Evaluator for this completion config. Populated by createChat/createModel.
+   * Internal; not part of the flag value shape.
+   */
+  evaluator?: Evaluator;
 }
 
 /**

packages/sdk/server-ai/src/api/judge/Evaluator.ts

@@ -0,0 +1,80 @@
+import { LDLogger } from '@launchdarkly/js-server-sdk-common';
+
+import { LDJudgeConfiguration } from '../config/types';
+import { Judge } from './Judge';
+import { LDJudgeResult } from './types';
+
+/**
+ * Wraps a collection of judges and a judge configuration, providing a single
+ * `evaluate` method that runs all configured judges against a given input/output pair.
+ *
+ * The `Evaluator` is responsible only for running judges and returning results.
+ * It does NOT call `tracker.trackJudgeResult` — that is the responsibility of
+ * the managed layer (e.g., ManagedModel, ManagedAgent).
+ */
+export class Evaluator {
+  constructor(
+    readonly judges: Map<string, Judge>,
+    readonly judgeConfiguration: LDJudgeConfiguration,
+    private readonly _logger?: LDLogger,
+  ) {}
+
+  /**
+   * Returns a no-op Evaluator that always resolves to an empty array.
+   * Use this when no judges are configured.
+   */
+  static noop(): Evaluator {
+    return new Evaluator(new Map(), { judges: [] });
+  }
+
+  /**
+   * Evaluates the given input/output pair using all configured judges.
+   * Missing judge instances are logged as warnings and skipped (not errors).
+   *
+   * @param input The input that was provided to the AI model.
+   * @param output The output produced by the AI model.
+   * @returns A promise resolving to an array of judge evaluation results.
+   */
+  async evaluate(input: string, output: string): Promise<LDJudgeResult[]> {
+    if (this.judgeConfiguration.judges.length === 0) {
+      return [];
+    }
+
+    const evaluationPromises = this.judgeConfiguration.judges.map(async (judgeConfig) => {
+      const judge = this.judges.get(judgeConfig.key);
+      if (!judge) {
+        this._logger?.warn(`Judge configuration is not enabled for ${judgeConfig.key}`);
+        // Skip — missing judge is a warning, not an error result
+        return null;
+      }
+
+      try {
+        return await judge.evaluate(input, output, judgeConfig.samplingRate);
+      } catch (err) {
+        const result: LDJudgeResult = {
+          success: false,
+          sampled: true,
+          errorMessage: err instanceof Error ? err.message : 'Unknown error',
+        };
+        return result;
+      }
+    });
+
+    const settled = await Promise.allSettled(evaluationPromises);
+
+    const results: LDJudgeResult[] = [];
+    settled.forEach((item) => {
+      if (item.status === 'fulfilled' && item.value !== null) {
+        results.push(item.value);
+      } else if (item.status === 'rejected') {
+        results.push({
+          success: false,
+          sampled: true,
+          errorMessage: 'Judge evaluation failed unexpectedly',
+        });
+      }
+    });
+
+    return results;
+  }
+}

packages/sdk/server-ai/src/api/judge/index.ts

@@ -1,2 +1,3 @@
 export { Judge } from './Judge';
+export { Evaluator } from './Evaluator';
 export type { LDJudgeResult, StructuredResponse } from './types';