feat: wire evaluations tracking chain in ManagedModel.run()

packages/sdk/server-ai/__tests__/ManagedModel.test.ts

@@ -1,13 +1,13 @@
 import { ManagedModel } from '../src/api/ManagedModel';
 import { LDAIConfigTracker } from '../src/api/config/LDAIConfigTracker';
 import { LDAICompletionConfig } from '../src/api/config/types';
-import { Evaluator } from '../src/api/judge/Evaluator';
 import { RunnerResult } from '../src/api/model/types';
 import { Runner } from '../src/api/providers/Runner';
 
 describe('ManagedModel', () => {
   let mockRunner: jest.Mocked<Runner>;
   let mockTracker: jest.Mocked<LDAIConfigTracker>;
+  let mockEvaluator: { evaluate: jest.Mock };
   let aiConfig: LDAICompletionConfig;
 
   beforeEach(() => {
@@ -32,14 +32,18 @@ describe('ManagedModel', () => {
       resumptionToken: 'resumption-token-123',
     } as any;
 
+    mockEvaluator = {
+      evaluate: jest.fn().mockResolvedValue([]),
+    };
+
     aiConfig = {
       key: 'test-config',
       enabled: true,
       messages: [{ role: 'system', content: 'You are a helpful assistant.' }],
       model: { name: 'gpt-4' },
       provider: { name: 'openai' },
       createTracker: () => mockTracker,
-      evaluator: Evaluator.noop(),
+      evaluator: mockEvaluator as any,
     };
   });
 

packages/sdk/server-ai/__tests__/ManagedModelRun.test.ts

@@ -0,0 +1,160 @@
+import { ManagedModel } from '../src/api/ManagedModel';
+import { LDAIConfigTracker } from '../src/api/config/LDAIConfigTracker';
+import { LDAICompletionConfig } from '../src/api/config/types';
+import { Evaluator } from '../src/api/judge/Evaluator';
+import { LDJudgeResult } from '../src/api/judge/types';
+import { RunnerResult } from '../src/api/model/types';
+import { Runner } from '../src/api/providers/Runner';
+
+describe('ManagedModel.run() evaluations', () => {
+  let mockRunner: jest.Mocked<Runner>;
+  let mockTracker: jest.Mocked<LDAIConfigTracker>;
+  let aiConfig: LDAICompletionConfig;
+
+  const runnerResult: RunnerResult = {
+    content: 'AI response content',
+    metrics: { success: true },
+  };
+
+  beforeEach(() => {
+    mockRunner = {
+      run: jest.fn().mockResolvedValue(runnerResult),
+    };
+
+    mockTracker = {
+      trackMetricsOf: jest.fn().mockImplementation(async (_extractor: any, func: any) => func()),
+      trackJudgeResult: jest.fn(),
+      resumptionToken: 'test-resumption-token',
+      getTrackData: jest.fn().mockReturnValue({}),
+      trackDuration: jest.fn(),
+      trackTokens: jest.fn(),
+      trackSuccess: jest.fn(),
+      trackError: jest.fn(),
+      trackFeedback: jest.fn(),
+      trackTimeToFirstToken: jest.fn(),
+      trackDurationOf: jest.fn(),
+      trackOpenAIMetrics: jest.fn(),
+      trackBedrockConverseMetrics: jest.fn(),
+      trackVercelAIMetrics: jest.fn(),
+      getSummary: jest
+        .fn()
+        .mockReturnValue({ success: true, resumptionToken: 'test-resumption-token' }),
+    } as any;
+
+    aiConfig = {
+      key: 'test-config',
+      enabled: true,
+      messages: [{ role: 'system', content: 'You are helpful.' }],
+      model: { name: 'gpt-4' },
+      provider: { name: 'openai' },
+      createTracker: () => mockTracker,
+      evaluator: Evaluator.noop(),
+    };
+  });
+
+  it('returns before evaluations resolve', async () => {
+    let resolveEval!: (v: LDJudgeResult[]) => void;
+    const slowEvaluator = {
+      judgeConfiguration: { judges: [{ key: 'judge-1', samplingRate: 1.0 }] },
+      evaluate: jest.fn().mockReturnValue(
+        new Promise<LDJudgeResult[]>((resolve) => {
+          resolveEval = resolve;
+        }),
+      ),
+      judges: new Map(),
+    } as unknown as Evaluator;
+
+    const configWithEvaluator: LDAICompletionConfig = {
+      ...aiConfig,
+      evaluator: slowEvaluator,
+    };
+
+    const model = new ManagedModel(configWithEvaluator, mockRunner);
+
+    let evaluationsResolved = false;
+    const result = await model.run('Hello');
+
+    expect(result.content).toBe('AI response content');
+
+    result.evaluations.then(() => {
+      evaluationsResolved = true;
+    });
+
+    await Promise.resolve();
+    expect(evaluationsResolved).toBe(false);
+
+    resolveEval([{ success: true, sampled: true, score: 0.9 }]);
+    await result.evaluations;
+    expect(evaluationsResolved).toBe(true);
+  });
+
+  it('awaiting evaluations guarantees tracking is complete', async () => {
+    const judgeResult: LDJudgeResult = {
+      success: true,
+      sampled: true,
+      score: 0.8,
+      metricKey: 'quality',
+    };
+    const mockEvaluator = {
+      judgeConfiguration: { judges: [{ key: 'judge-1', samplingRate: 1.0 }] },
+      evaluate: jest.fn().mockResolvedValue([judgeResult]),
+      judges: new Map(),
+    } as unknown as Evaluator;
+
+    const configWithEvaluator: LDAICompletionConfig = {
+      ...aiConfig,
+      evaluator: mockEvaluator,
+    };
+
+    const model = new ManagedModel(configWithEvaluator, mockRunner);
+    const result = await model.run('Hello');
+
+    await result.evaluations;
+    expect(mockTracker.trackJudgeResult).toHaveBeenCalledWith(judgeResult);
+  });
+
+  it('builds ManagedResult with correct content and metrics', async () => {
+    const model = new ManagedModel(aiConfig, mockRunner);
+    const result = await model.run('test prompt');
+
+    expect(result.content).toBe('AI response content');
+    expect(result.metrics.success).toBe(true);
+    expect(result.metrics.resumptionToken).toBe('test-resumption-token');
+    expect(result.evaluations).toBeInstanceOf(Promise);
+  });
+
+  it('resolves to empty evaluations when evaluator is noop', async () => {
+    const configWithNoop: LDAICompletionConfig = {
+      ...aiConfig,
+      evaluator: Evaluator.noop(),
+    };
+    const model = new ManagedModel(configWithNoop, mockRunner);
+    const result = await model.run('Hello');
+    const evaluations = await result.evaluations;
+    expect(evaluations).toEqual([]);
+  });
+
+  it('passes the prompt to evaluator.evaluate as input', async () => {
+    const judgeResult: LDJudgeResult = {
+      success: true,
+      sampled: true,
+      score: 1.0,
+    };
+    const mockEvaluator = {
+      judgeConfiguration: { judges: [{ key: 'judge-1', samplingRate: 1.0 }] },
+      evaluate: jest.fn().mockResolvedValue([judgeResult]),
+      judges: new Map(),
+    } as unknown as Evaluator;
+
+    const configWithEvaluator: LDAICompletionConfig = {
+      ...aiConfig,
+      evaluator: mockEvaluator,
+    };
+
+    const model = new ManagedModel(configWithEvaluator, mockRunner);
+    const result = await model.run('user prompt here');
+    await result.evaluations;
+
+    expect(mockEvaluator.evaluate).toHaveBeenCalledWith('user prompt here', 'AI response content');
+  });
+});

packages/sdk/server-ai/src/api/ManagedModel.ts

@@ -1,13 +1,12 @@
 import { LDLogger } from '@launchdarkly/js-server-sdk-common';
 
 import { LDAICompletionConfig } from './config/types';
-import { LDJudgeResult } from './judge/types';
 import { ManagedResult, RunnerResult } from './model/types';
 import { Runner } from './providers/Runner';
 
 /**
  * ManagedModel provides chat-completion invocation with automatic tracking and
- * (in a future PR) automatic judge evaluation.
+ * automatic judge evaluation.
  *
  * The class is stateless: each `run()` call sends the prompt directly to the
  * underlying `Runner` and returns a `ManagedResult`. Conversation history,
@@ -42,11 +41,16 @@ export class ManagedModel {
 
     const metrics = tracker.getSummary();
 
-    // Evaluations are wired in a follow-up PR. For now, resolve empty.
-    const evaluations: Promise<LDJudgeResult[]> = Promise.resolve([]);
+    const output = result.content;
+    const evaluations = this.aiConfig.evaluator.evaluate(prompt, output).then((results) => {
+      results.forEach((judgeResult) => {
+        tracker.trackJudgeResult(judgeResult);
+      });
+      return results;
+    });
 
     return {
-      content: result.content,
+      content: output,
       metrics,
       raw: result.raw,
       parsed: result.parsed,