feat: add ManagedAgent with evaluations support (AIC-1657)

Introduces ManagedAgent class as the agent-mode analogue of ManagedModel.
ManagedAgent.run() invokes the runner, tracks metrics, and wires judge
evaluations (via aiAgentConfig.evaluator) into ManagedResult.evaluations.
Adds LDAIClient.createAgent() to LDAIClientImpl and the LDAIClient interface.
Exports ManagedAgent from the public API.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: jsonbailey

packages/sdk/server-ai/__tests__/ManagedAgent.test.ts

@@ -0,0 +1,170 @@
+import { ManagedAgent } from '../src/api/agent/ManagedAgent';
+import { LDAIConfigTracker } from '../src/api/config/LDAIConfigTracker';
+import { LDAIAgentConfig } from '../src/api/config/types';
+import { Evaluator } from '../src/api/judge/Evaluator';
+import { LDJudgeResult } from '../src/api/judge/types';
+import { RunnerResult } from '../src/api/model/types';
+import { Runner } from '../src/api/providers/Runner';
+
+describe('ManagedAgent', () => {
+  let mockRunner: jest.Mocked<Runner>;
+  let mockTracker: jest.Mocked<LDAIConfigTracker>;
+  let agentConfig: LDAIAgentConfig;
+
+  const runnerResult: RunnerResult = {
+    content: 'Agent response',
+    metrics: { success: true },
+  };
+
+  beforeEach(() => {
+    mockRunner = {
+      run: jest.fn().mockResolvedValue(runnerResult),
+    };
+
+    mockTracker = {
+      trackMetricsOf: jest.fn().mockImplementation(async (_extractor: any, func: any) => func()),
+      trackJudgeResult: jest.fn(),
+      resumptionToken: 'agent-resumption-token',
+      getTrackData: jest.fn().mockReturnValue({}),
+      trackDuration: jest.fn(),
+      trackTokens: jest.fn(),
+      trackSuccess: jest.fn(),
+      trackError: jest.fn(),
+      trackFeedback: jest.fn(),
+      trackTimeToFirstToken: jest.fn(),
+      trackDurationOf: jest.fn(),
+      trackOpenAIMetrics: jest.fn(),
+      trackBedrockConverseMetrics: jest.fn(),
+      trackVercelAIMetrics: jest.fn(),
+      getSummary: jest.fn(),
+    } as any;
+
+    agentConfig = {
+      key: 'test-agent',
+      enabled: true,
+      instructions: 'You are a helpful agent.',
+      model: { name: 'gpt-4' },
+      provider: { name: 'openai' },
+      createTracker: () => mockTracker,
+    };
+  });
+
+  it('returns a ManagedResult with content and metrics', async () => {
+    const agent = new ManagedAgent(agentConfig, mockRunner);
+    const result = await agent.run('Hello agent');
+
+    expect(result.content).toBe('Agent response');
+    expect(result.metrics.success).toBe(true);
+    expect(result.metrics.resumptionToken).toBe('agent-resumption-token');
+  });
+
+  it('passes the prompt directly to the runner', async () => {
+    const agent = new ManagedAgent(agentConfig, mockRunner);
+    await agent.run('My question');
+
+    expect(mockRunner.run).toHaveBeenCalledWith('My question');
+  });
+
+  it('resolves to empty evaluations when no evaluator configured', async () => {
+    const agent = new ManagedAgent(agentConfig, mockRunner);
+    const result = await agent.run('Hello');
+    const evaluations = await result.evaluations;
+    expect(evaluations).toEqual([]);
+  });
+
+  it('resolves to empty evaluations with noop evaluator', async () => {
+    const configWithNoop: LDAIAgentConfig = {
+      ...agentConfig,
+      evaluator: Evaluator.noop(),
+    };
+    const agent = new ManagedAgent(configWithNoop, mockRunner);
+    const result = await agent.run('Hello');
+    const evaluations = await result.evaluations;
+    expect(evaluations).toEqual([]);
+  });
+
+  it('awaiting evaluations calls tracker.trackJudgeResult', async () => {
+    const judgeResult: LDJudgeResult = {
+      success: true,
+      sampled: true,
+      score: 0.85,
+      metricKey: 'quality',
+    };
+    const mockEvaluator = {
+      judgeConfiguration: { judges: [{ key: 'judge-1', samplingRate: 1.0 }] },
+      evaluate: jest.fn().mockResolvedValue([judgeResult]),
+      judges: new Map(),
+    } as unknown as Evaluator;
+
+    const configWithEvaluator: LDAIAgentConfig = {
+      ...agentConfig,
+      evaluator: mockEvaluator,
+    };
+
+    const agent = new ManagedAgent(configWithEvaluator, mockRunner);
+    const result = await agent.run('Hello');
+
+    await result.evaluations;
+    expect(mockTracker.trackJudgeResult).toHaveBeenCalledWith(judgeResult);
+  });
+
+  it('passes the prompt to evaluator.evaluate as input', async () => {
+    const mockEvaluator = {
+      judgeConfiguration: { judges: [{ key: 'judge-1', samplingRate: 1.0 }] },
+      evaluate: jest.fn().mockResolvedValue([]),
+      judges: new Map(),
+    } as unknown as Evaluator;
+
+    const configWithEvaluator: LDAIAgentConfig = {
+      ...agentConfig,
+      evaluator: mockEvaluator,
+    };
+
+    const agent = new ManagedAgent(configWithEvaluator, mockRunner);
+    const result = await agent.run('user prompt');
+    await result.evaluations;
+
+    expect(mockEvaluator.evaluate).toHaveBeenCalledWith('user prompt', 'Agent response');
+  });
+
+  it('returns before evaluations resolve', async () => {
+    let resolveEval!: (v: LDJudgeResult[]) => void;
+    const slowEvaluator = {
+      judgeConfiguration: { judges: [{ key: 'judge-1', samplingRate: 1.0 }] },
+      evaluate: jest.fn().mockReturnValue(
+        new Promise<LDJudgeResult[]>((resolve) => {
+          resolveEval = resolve;
+        }),
+      ),
+      judges: new Map(),
+    } as unknown as Evaluator;
+
+    const configWithEvaluator: LDAIAgentConfig = {
+      ...agentConfig,
+      evaluator: slowEvaluator,
+    };
+
+    const agent = new ManagedAgent(configWithEvaluator, mockRunner);
+
+    let evaluationsResolved = false;
+    const result = await agent.run('Hello');
+
+    expect(result.content).toBe('Agent response');
+
+    result.evaluations.then(() => {
+      evaluationsResolved = true;
+    });
+
+    await Promise.resolve();
+    expect(evaluationsResolved).toBe(false);
+
+    resolveEval([{ success: true, sampled: true, score: 0.9 }]);
+    await result.evaluations;
+    expect(evaluationsResolved).toBe(true);
+  });
+
+  it('exposes the agent config via getConfig', () => {
+    const agent = new ManagedAgent(agentConfig, mockRunner);
+    expect(agent.getConfig()).toBe(agentConfig);
+  });
+});

packages/sdk/server-ai/src/LDAIClientImpl.ts

@@ -3,6 +3,7 @@ import { randomUUID } from 'node:crypto';
 
 import { LDContext, LDLogger } from '@launchdarkly/js-server-sdk-common';
 
+import { ManagedAgent } from './api/agent/ManagedAgent';
 import { ManagedModel } from './api/chat';
 import {
   LDAIAgentConfig,
@@ -38,6 +39,7 @@ import { aiSdkLanguage, aiSdkName, aiSdkVersion } from './sdkInfo';
 const TRACK_SDK_INFO = '$ld:ai:sdk:info';
 const TRACK_USAGE_COMPLETION_CONFIG = '$ld:ai:usage:completion-config';
 const TRACK_USAGE_CREATE_CHAT = '$ld:ai:usage:create-chat';
+const TRACK_USAGE_CREATE_AGENT = '$ld:ai:usage:create-agent';
 const TRACK_USAGE_JUDGE_CONFIG = '$ld:ai:usage:judge-config';
 const TRACK_USAGE_CREATE_JUDGE = '$ld:ai:usage:create-judge';
 const TRACK_USAGE_AGENT_CONFIG = '$ld:ai:usage:agent-config';
@@ -72,6 +74,30 @@ function runnerFromAIProvider(provider: AIProvider, config: LDAICompletionConfig
   };
 }
 
+/**
+ * Adapt a (deprecated) AIProvider to the Runner protocol for agent configs.
+ * Prepends the agent's instructions as a system message so existing
+ * AIProvider-based agent flows preserve their instruction behavior under the
+ * stateless Runner contract.
+ */
+function runnerFromAIProviderForAgent(provider: AIProvider, config: LDAIAgentConfig): Runner {
+  return {
+    async run(input: string): Promise<RunnerResult> {
+      const messages: LDMessage[] = [];
+      if (config.instructions) {
+        messages.push({ role: 'system', content: config.instructions });
+      }
+      messages.push({ role: 'user', content: input });
+      const response = await provider.invokeModel(messages);
+      return {
+        content: response.message.content,
+        metrics: response.metrics,
+        raw: response,
+      };
+    },
+  };
+}
+
 export class LDAIClientImpl implements LDAIClient {
   private _logger?: LDLogger;
 
@@ -439,6 +465,45 @@ export class LDAIClientImpl implements LDAIClient {
     return new ManagedModel(configWithEvaluator, runner, this._logger);
   }
 
+  async createAgent(
+    key: string,
+    context: LDContext,
+    defaultValue?: LDAIAgentConfigDefault,
+    variables?: Record<string, unknown>,
+    defaultAiProvider?: SupportedAIProvider,
+  ): Promise<ManagedAgent | undefined> {
+    this._ldClient.track(TRACK_USAGE_CREATE_AGENT, context, key, 1);
+
+    const config = await this._agentConfig(
+      key,
+      context,
+      defaultValue ?? disabledAIConfig,
+      variables,
+    );
+
+    if (!config.enabled) {
+      this._logger?.info(`Agent configuration is disabled: ${key}`);
+      return undefined;
+    }
+
+    const provider = await AIProviderFactory.create(config, this._logger, defaultAiProvider);
+    if (!provider) {
+      return undefined;
+    }
+
+    const evaluator = await this._buildEvaluator(
+      config.judgeConfiguration?.judges ?? [],
+      context,
+      variables,
+      defaultAiProvider,
+    );
+
+    const configWithEvaluator: LDAIAgentConfig = { ...config, evaluator };
+
+    const runner = runnerFromAIProviderForAgent(provider, configWithEvaluator);
+    return new ManagedAgent(configWithEvaluator, runner, this._logger);
+  }
+
   /**
    * @deprecated Use `createModel` instead. This method will be removed in a future version.
    */

packages/sdk/server-ai/src/api/LDAIClient.ts

@@ -1,5 +1,6 @@
 import { LDContext } from '@launchdarkly/js-server-sdk-common';
 
+import { ManagedAgent } from './agent/ManagedAgent';
 import { ManagedModel } from './chat';
 import {
   LDAIAgentConfig,
@@ -275,6 +276,25 @@ export interface LDAIClient {
     defaultAiProvider?: SupportedAIProvider,
   ): Promise<ManagedModel | undefined>;
 
+  /**
+   * Creates and returns a new ManagedAgent instance for agent interactions.
+   * Evaluations are wired automatically and exposed on ManagedResult.evaluations.
+   *
+   * @param key The key identifying the agent AI config to use.
+   * @param context The standard LDContext used when evaluating flags.
+   * @param defaultValue Optional fallback when the configuration is not available from LaunchDarkly.
+   * @param variables Dictionary of values for instruction interpolation.
+   * @param defaultAiProvider Optional default AI provider to use.
+   * @returns A promise that resolves to the ManagedAgent instance, or undefined if disabled.
+   */
+  createAgent(
+    key: string,
+    context: LDContext,
+    defaultValue?: LDAIAgentConfigDefault,
+    variables?: Record<string, unknown>,
+    defaultAiProvider?: SupportedAIProvider,
+  ): Promise<ManagedAgent | undefined>;
+
   /**
    * @deprecated Use `createModel` instead. This method will be removed in a future version.
    */

packages/sdk/server-ai/src/api/agent/ManagedAgent.ts

@@ -0,0 +1,80 @@
+import { LDLogger } from '@launchdarkly/js-server-sdk-common';
+
+import { LDAIAgentConfig } from '../config/types';
+import { LDJudgeResult } from '../judge/types';
+import { LDAIMetricSummary, ManagedResult, RunnerResult } from '../model/types';
+import { Runner } from '../providers/Runner';
+
+/**
+ * ManagedAgent provides agent invocation with automatic tracking and automatic
+ * judge evaluation.
+ *
+ * The class is stateless: each `run()` call sends the prompt directly to the
+ * underlying `Runner` and returns a `ManagedResult`. Conversation history,
+ * if any, must be managed by the caller (or by the Runner implementation).
+ *
+ * Obtain an instance via `LDAIClient.createAgent()`.
+ */
+export class ManagedAgent {
+  constructor(
+    protected readonly aiAgentConfig: LDAIAgentConfig,
+    protected readonly runner: Runner,
+    private readonly _logger?: LDLogger,
+  ) {}
+
+  /**
+   * Invoke the agent with a prompt string and return a ManagedResult.
+   *
+   * `run()` resolves before `ManagedResult.evaluations` resolves. Awaiting
+   * `evaluations` guarantees both judge evaluation and tracker.trackJudgeResult()
+   * are complete.
+   *
+   * @param prompt The user input to send to the agent.
+   * @returns Promise resolving to ManagedResult (before evaluations settle).
+   */
+  async run(prompt: string): Promise<ManagedResult> {
+    const tracker = this.aiAgentConfig.createTracker!();
+
+    const result = await tracker.trackMetricsOf(
+      (r: RunnerResult) => r.metrics,
+      () => this.runner.run(prompt),
+    );
+
+    const metrics: LDAIMetricSummary = {
+      success: result.metrics.success,
+      usage: result.metrics.usage,
+      toolCalls: result.metrics.toolCalls,
+      durationMs: result.metrics.durationMs,
+      resumptionToken: tracker.resumptionToken,
+    };
+
+    const output = result.content;
+    const evaluator = this.aiAgentConfig.evaluator;
+    let evaluations: Promise<LDJudgeResult[]>;
+    if (evaluator) {
+      evaluations = evaluator.evaluate(prompt, output).then((results) => {
+        results.forEach((judgeResult) => {
+          tracker.trackJudgeResult(judgeResult);
+        });
+        return results;
+      });
+    } else {
+      evaluations = Promise.resolve([]);
+    }
+
+    return {
+      content: output,
+      metrics,
+      raw: result.raw,
+      parsed: result.parsed,
+      evaluations,
+    };
+  }
+
+  /**
+   * Get the underlying AI agent configuration used to initialize this ManagedAgent.
+   */
+  getConfig(): LDAIAgentConfig {
+    return this.aiAgentConfig;
+  }
+}

packages/sdk/server-ai/src/api/agent/index.ts

@@ -0,0 +1 @@
+export { ManagedAgent } from './ManagedAgent';

packages/sdk/server-ai/src/api/index.ts

@@ -1,4 +1,5 @@
 export * from './config';
+export * from './agent';
 export * from './chat';
 export * from './graph';
 export * from './judge';