feat: add ManagedAgent with evaluations support (AIC-1657)

Introduces ManagedAgent class as the agent-mode analogue of TrackedChat.
ManagedAgent.run() invokes the provider, tracks metrics, and wires judge
evaluations (via aiAgentConfig.evaluator) into ManagedResult.evaluations.
Adds LDAIClient.createAgent() to LDAIClientImpl and the LDAIClient interface.
Exports ManagedAgent from the public API.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: jsonbailey

packages/sdk/server-ai/__tests__/ManagedAgent.test.ts

@@ -0,0 +1,144 @@
+import { ManagedAgent } from '../src/api/agent/ManagedAgent';
+import { LDAIAgentConfig } from '../src/api/config/types';
+import { LDAIConfigTracker } from '../src/api/config/LDAIConfigTracker';
+import { Evaluator } from '../src/api/judge/Evaluator';
+import { LDJudgeResult } from '../src/api/judge/types';
+import { AIProvider } from '../src/api/providers/AIProvider';
+import { ChatResponse } from '../src/api/chat/types';
+
+describe('ManagedAgent', () => {
+  let mockProvider: jest.Mocked<AIProvider>;
+  let mockTracker: jest.Mocked<LDAIConfigTracker>;
+  let agentConfig: LDAIAgentConfig;
+
+  const mockResponse: ChatResponse = {
+    message: { role: 'assistant', content: 'Agent response' },
+    metrics: { success: true },
+  };
+
+  beforeEach(() => {
+    mockProvider = {
+      invokeModel: jest.fn().mockResolvedValue(mockResponse),
+    } as any;
+
+    mockTracker = {
+      trackMetricsOf: jest.fn().mockImplementation(async (_extractor: any, func: any) => func()),
+      trackJudgeResult: jest.fn(),
+      resumptionToken: 'agent-resumption-token',
+      getTrackData: jest.fn().mockReturnValue({}),
+      trackDuration: jest.fn(),
+      trackTokens: jest.fn(),
+      trackSuccess: jest.fn(),
+      trackError: jest.fn(),
+      trackFeedback: jest.fn(),
+      trackTimeToFirstToken: jest.fn(),
+      trackDurationOf: jest.fn(),
+      trackOpenAIMetrics: jest.fn(),
+      trackBedrockConverseMetrics: jest.fn(),
+      trackVercelAISDKGenerateTextMetrics: jest.fn(),
+      trackStreamMetricsOf: jest.fn(),
+      trackToolCall: jest.fn(),
+      trackToolCalls: jest.fn(),
+      getSummary: jest.fn(),
+    } as any;
+
+    agentConfig = {
+      key: 'test-agent',
+      enabled: true,
+      instructions: 'You are a helpful agent.',
+      model: { name: 'gpt-4' },
+      provider: { name: 'openai' },
+      createTracker: () => mockTracker,
+    };
+  });
+
+  it('run() returns a ManagedResult with content and metrics', async () => {
+    const agent = new ManagedAgent(agentConfig, mockProvider);
+    const result = await agent.run('Hello agent');
+
+    expect(result.content).toBe('Agent response');
+    expect(result.metrics.success).toBe(true);
+    expect(result.metrics.resumptionToken).toBe('agent-resumption-token');
+  });
+
+  it('run() invokes the provider with the prompt as user message', async () => {
+    const agent = new ManagedAgent(agentConfig, mockProvider);
+    await agent.run('My question');
+
+    expect(mockProvider.invokeModel).toHaveBeenCalledWith([
+      { role: 'user', content: 'My question' },
+    ]);
+  });
+
+  it('run() resolves to empty evaluations when no evaluator configured', async () => {
+    const agent = new ManagedAgent(agentConfig, mockProvider);
+    const result = await agent.run('Hello');
+    const evaluations = await result.evaluations;
+    expect(evaluations).toEqual([]);
+  });
+
+  it('run() resolves to empty evaluations with noop evaluator', async () => {
+    const configWithNoop: LDAIAgentConfig = {
+      ...agentConfig,
+      evaluator: Evaluator.noop(),
+    };
+    const agent = new ManagedAgent(configWithNoop, mockProvider);
+    const result = await agent.run('Hello');
+    const evaluations = await result.evaluations;
+    expect(evaluations).toEqual([]);
+  });
+
+  it('awaiting evaluations calls tracker.trackJudgeResult', async () => {
+    const judgeResult: LDJudgeResult = {
+      success: true,
+      sampled: true,
+      score: 0.85,
+      metricKey: 'quality',
+    };
+    const mockEvaluator = {
+      judgeConfiguration: { judges: [{ key: 'judge-1', samplingRate: 1.0 }] },
+      evaluate: jest.fn().mockResolvedValue([judgeResult]),
+      judges: new Map(),
+    } as unknown as Evaluator;
+
+    const configWithEvaluator: LDAIAgentConfig = {
+      ...agentConfig,
+      evaluator: mockEvaluator,
+    };
+
+    const agent = new ManagedAgent(configWithEvaluator, mockProvider);
+    const result = await agent.run('Hello');
+
+    await result.evaluations;
+    expect(mockTracker.trackJudgeResult).toHaveBeenCalledWith(judgeResult);
+  });
+
+  it('evaluate() is called with prompt as input and response content as output', async () => {
+    const mockEvaluator = {
+      judgeConfiguration: { judges: [{ key: 'judge-1', samplingRate: 1.0 }] },
+      evaluate: jest.fn().mockResolvedValue([]),
+      judges: new Map(),
+    } as unknown as Evaluator;
+
+    const configWithEvaluator: LDAIAgentConfig = {
+      ...agentConfig,
+      evaluator: mockEvaluator,
+    };
+
+    const agent = new ManagedAgent(configWithEvaluator, mockProvider);
+    const result = await agent.run('user prompt');
+    await result.evaluations;
+
+    expect(mockEvaluator.evaluate).toHaveBeenCalledWith('user prompt', 'Agent response');
+  });
+
+  it('getConfig() returns the agent config', () => {
+    const agent = new ManagedAgent(agentConfig, mockProvider);
+    expect(agent.getConfig()).toBe(agentConfig);
+  });
+
+  it('getProvider() returns the provider', () => {
+    const agent = new ManagedAgent(agentConfig, mockProvider);
+    expect(agent.getProvider()).toBe(mockProvider);
+  });
+});

packages/sdk/server-ai/src/LDAIClientImpl.ts

@@ -3,6 +3,7 @@ import { randomUUID } from 'node:crypto';
 
 import { LDContext, LDLogger } from '@launchdarkly/js-server-sdk-common';
 
+import { ManagedAgent } from './api/agent/ManagedAgent';
 import { TrackedChat } from './api/chat';
 import {
   LDAIAgentConfig,
@@ -37,6 +38,7 @@ import { aiSdkLanguage, aiSdkName, aiSdkVersion } from './sdkInfo';
 const TRACK_SDK_INFO = '$ld:ai:sdk:info';
 const TRACK_USAGE_COMPLETION_CONFIG = '$ld:ai:usage:completion-config';
 const TRACK_USAGE_CREATE_CHAT = '$ld:ai:usage:create-chat';
+const TRACK_USAGE_CREATE_AGENT = '$ld:ai:usage:create-agent';
 const TRACK_USAGE_JUDGE_CONFIG = '$ld:ai:usage:judge-config';
 const TRACK_USAGE_CREATE_JUDGE = '$ld:ai:usage:create-judge';
 const TRACK_USAGE_AGENT_CONFIG = '$ld:ai:usage:agent-config';
@@ -420,6 +422,44 @@ export class LDAIClientImpl implements LDAIClient {
     return this.createChat(key, context, defaultValue, variables, defaultAiProvider);
   }
 
+  async createAgent(
+    key: string,
+    context: LDContext,
+    defaultValue?: LDAIAgentConfigDefault,
+    variables?: Record<string, unknown>,
+    defaultAiProvider?: SupportedAIProvider,
+  ): Promise<ManagedAgent | undefined> {
+    this._ldClient.track(TRACK_USAGE_CREATE_AGENT, context, key, 1);
+
+    const config = await this._agentConfig(
+      key,
+      context,
+      defaultValue ?? disabledAIConfig,
+      variables,
+    );
+
+    if (!config.enabled) {
+      this._logger?.info(`Agent configuration is disabled: ${key}`);
+      return undefined;
+    }
+
+    const provider = await AIProviderFactory.create(config, this._logger, defaultAiProvider);
+    if (!provider) {
+      return undefined;
+    }
+
+    const evaluator = await this._buildEvaluator(
+      config.judgeConfiguration?.judges ?? [],
+      context,
+      variables,
+      defaultAiProvider,
+    );
+
+    const configWithEvaluator: LDAIAgentConfig = { ...config, evaluator };
+
+    return new ManagedAgent(configWithEvaluator, provider, this._logger);
+  }
+
   /**
    * @deprecated Use `createChat` instead. This method will be removed in a future version.
    */

packages/sdk/server-ai/src/api/LDAIClient.ts

@@ -1,5 +1,6 @@
 import { LDContext } from '@launchdarkly/js-server-sdk-common';
 
+import { ManagedAgent } from './agent/ManagedAgent';
 import { TrackedChat } from './chat';
 import {
   LDAIAgentConfig,
@@ -296,6 +297,25 @@ export interface LDAIClient {
     defaultAiProvider?: SupportedAIProvider,
   ): Promise<TrackedChat | undefined>;
 
+  /**
+   * Creates and returns a new ManagedAgent instance for agent interactions.
+   * Evaluations are wired automatically and exposed on ManagedResult.evaluations.
+   *
+   * @param key The key identifying the agent AI config to use.
+   * @param context The standard LDContext used when evaluating flags.
+   * @param defaultValue Optional fallback when the configuration is not available from LaunchDarkly.
+   * @param variables Dictionary of values for instruction interpolation.
+   * @param defaultAiProvider Optional default AI provider to use.
+   * @returns A promise that resolves to the ManagedAgent instance, or undefined if disabled.
+   */
+  createAgent(
+    key: string,
+    context: LDContext,
+    defaultValue?: LDAIAgentConfigDefault,
+    variables?: Record<string, unknown>,
+    defaultAiProvider?: SupportedAIProvider,
+  ): Promise<ManagedAgent | undefined>;
+
   /**
    * @deprecated Use `createChat` instead. This method will be removed in a future version.
    */

packages/sdk/server-ai/src/api/agent/ManagedAgent.ts

@@ -0,0 +1,93 @@
+import { LDLogger } from '@launchdarkly/js-server-sdk-common';
+
+import { LDAIAgentConfig } from '../config/types';
+import { LDJudgeResult } from '../judge/types';
+import { LDAIMetricSummary, ManagedResult } from '../model/types';
+import { AIProvider } from '../providers/AIProvider';
+import { ChatResponse } from '../chat/types';
+
+/**
+ * ManagedAgent provides agent invocation with automatic judge evaluation.
+ *
+ * This is the agent-mode analogue of TrackedChat (ManagedModel). It invokes
+ * the provider, tracks metrics, and wires judge evaluations into a single
+ * Promise exposed on ManagedResult.evaluations.
+ *
+ * Obtain an instance via `LDAIClient.createAgent()`.
+ */
+export class ManagedAgent {
+  constructor(
+    protected readonly aiAgentConfig: LDAIAgentConfig,
+    protected readonly provider: AIProvider,
+    private readonly _logger?: LDLogger,
+  ) {}
+
+  /**
+   * Invoke the agent with a prompt string and return a ManagedResult.
+   *
+   * run() returns before ManagedResult.evaluations resolves. Awaiting evaluations
+   * guarantees both evaluation and tracker.trackJudgeResult() are complete.
+   *
+   * @param prompt The user input to send to the agent.
+   * @returns Promise resolving to ManagedResult (before evaluations settle).
+   */
+  async run(prompt: string): Promise<ManagedResult> {
+    const tracker = this.aiAgentConfig.createTracker!();
+
+    const userMessage = { role: 'user' as const, content: prompt };
+    const allMessages = [userMessage];
+
+    // Delegate to provider-specific implementation with tracking
+    const response = await tracker.trackMetricsOf(
+      (result: ChatResponse) => result.metrics,
+      () => this.provider.invokeModel(allMessages),
+    );
+
+    // Build the metric summary from response metrics + resumption token
+    const metrics: LDAIMetricSummary = {
+      success: response.metrics.success,
+      usage: response.metrics.usage,
+      toolCalls: response.metrics.toolCalls,
+      durationMs: response.metrics.durationMs,
+      resumptionToken: tracker.resumptionToken,
+    };
+
+    const output = response.message.content;
+
+    // Wire evaluation + tracking into a single Promise.
+    // run() returns before this resolves — awaiting evaluations guarantees
+    // both evaluation and tracking are complete.
+    const evaluator = this.aiAgentConfig.evaluator;
+    let evaluations: Promise<LDJudgeResult[]>;
+    if (evaluator && evaluator.judgeConfiguration.judges.length > 0) {
+      evaluations = evaluator.evaluate(prompt, output).then((results) => {
+        results.forEach((judgeResult) => {
+          tracker.trackJudgeResult(judgeResult);
+        });
+        return results;
+      });
+    } else {
+      evaluations = Promise.resolve([]);
+    }
+
+    return {
+      content: output,
+      metrics,
+      evaluations,
+    };
+  }
+
+  /**
+   * Get the underlying AI agent configuration.
+   */
+  getConfig(): LDAIAgentConfig {
+    return this.aiAgentConfig;
+  }
+
+  /**
+   * Get the underlying AI provider instance.
+   */
+  getProvider(): AIProvider {
+    return this.provider;
+  }
+}

packages/sdk/server-ai/src/api/agent/index.ts

@@ -0,0 +1 @@
+export { ManagedAgent } from './ManagedAgent';

packages/sdk/server-ai/src/api/index.ts

@@ -1,4 +1,5 @@
 export * from './config';
+export * from './agent';
 export * from './chat';
 export * from './graph';
 export * from './judge';