feat: add ManagedAgent with evaluations support

packages/sdk/server-ai/__tests__/ManagedAgent.test.ts

@@ -0,0 +1,144 @@
+import { ManagedAgent } from '../src/api/agent/ManagedAgent';
+import { ChatResponse } from '../src/api/chat/types';
+import { LDAIConfigTracker } from '../src/api/config/LDAIConfigTracker';
+import { LDAIAgentConfig } from '../src/api/config/types';
+import { Evaluator } from '../src/api/judge/Evaluator';
+import { LDJudgeResult } from '../src/api/judge/types';
+import { AIProvider } from '../src/api/providers/AIProvider';
+
+describe('ManagedAgent', () => {
+  let mockProvider: jest.Mocked<AIProvider>;
+  let mockTracker: jest.Mocked<LDAIConfigTracker>;
+  let agentConfig: LDAIAgentConfig;
+
+  const mockResponse: ChatResponse = {
+    message: { role: 'assistant', content: 'Agent response' },
+    metrics: { success: true },
+  };
+
+  beforeEach(() => {
+    mockProvider = {
+      invokeModel: jest.fn().mockResolvedValue(mockResponse),
+    } as any;
+
+    mockTracker = {
+      trackMetricsOf: jest.fn().mockImplementation(async (_extractor: any, func: any) => func()),
+      trackJudgeResult: jest.fn(),
+      resumptionToken: 'agent-resumption-token',
+      getTrackData: jest.fn().mockReturnValue({}),
+      trackDuration: jest.fn(),
+      trackTokens: jest.fn(),
+      trackSuccess: jest.fn(),
+      trackError: jest.fn(),
+      trackFeedback: jest.fn(),
+      trackTimeToFirstToken: jest.fn(),
+      trackDurationOf: jest.fn(),
+      trackOpenAIMetrics: jest.fn(),
+      trackBedrockConverseMetrics: jest.fn(),
+      trackVercelAISDKGenerateTextMetrics: jest.fn(),
+      trackStreamMetricsOf: jest.fn(),
+      trackToolCall: jest.fn(),
+      trackToolCalls: jest.fn(),
+      getSummary: jest.fn(),
+    } as any;
+
+    agentConfig = {
+      key: 'test-agent',
+      enabled: true,
+      instructions: 'You are a helpful agent.',
+      model: { name: 'gpt-4' },
+      provider: { name: 'openai' },
+      createTracker: () => mockTracker,
+    };
+  });
+
+  it('run() returns a ManagedResult with content and metrics', async () => {
+    const agent = new ManagedAgent(agentConfig, mockProvider);
+    const result = await agent.run('Hello agent');
+
+    expect(result.content).toBe('Agent response');
+    expect(result.metrics.success).toBe(true);
+    expect(result.metrics.resumptionToken).toBe('agent-resumption-token');
+  });
+
+  it('run() invokes the provider with the prompt as user message', async () => {
+    const agent = new ManagedAgent(agentConfig, mockProvider);
+    await agent.run('My question');
+
+    expect(mockProvider.invokeModel).toHaveBeenCalledWith([
+      { role: 'user', content: 'My question' },
+    ]);
+  });
+
+  it('run() resolves to empty evaluations when no evaluator configured', async () => {
+    const agent = new ManagedAgent(agentConfig, mockProvider);
+    const result = await agent.run('Hello');
+    const evaluations = await result.evaluations;
+    expect(evaluations).toEqual([]);
+  });
+
+  it('run() resolves to empty evaluations with noop evaluator', async () => {
+    const configWithNoop: LDAIAgentConfig = {
+      ...agentConfig,
+      evaluator: Evaluator.noop(),
+    };
+    const agent = new ManagedAgent(configWithNoop, mockProvider);
+    const result = await agent.run('Hello');
+    const evaluations = await result.evaluations;
+    expect(evaluations).toEqual([]);
+  });
+
+  it('awaiting evaluations calls tracker.trackJudgeResult', async () => {
+    const judgeResult: LDJudgeResult = {
+      success: true,
+      sampled: true,
+      score: 0.85,
+      metricKey: 'quality',
+    };
+    const mockEvaluator = {
+      judgeConfiguration: { judges: [{ key: 'judge-1', samplingRate: 1.0 }] },
+      evaluate: jest.fn().mockResolvedValue([judgeResult]),
+      judges: new Map(),
+    } as unknown as Evaluator;
+
+    const configWithEvaluator: LDAIAgentConfig = {
+      ...agentConfig,
+      evaluator: mockEvaluator,
+    };
+
+    const agent = new ManagedAgent(configWithEvaluator, mockProvider);
+    const result = await agent.run('Hello');
+
+    await result.evaluations;
+    expect(mockTracker.trackJudgeResult).toHaveBeenCalledWith(judgeResult);
+  });
+
+  it('evaluate() is called with prompt as input and response content as output', async () => {
+    const mockEvaluator = {
+      judgeConfiguration: { judges: [{ key: 'judge-1', samplingRate: 1.0 }] },
+      evaluate: jest.fn().mockResolvedValue([]),
+      judges: new Map(),
+    } as unknown as Evaluator;
+
+    const configWithEvaluator: LDAIAgentConfig = {
+      ...agentConfig,
+      evaluator: mockEvaluator,
+    };
+
+    const agent = new ManagedAgent(configWithEvaluator, mockProvider);
+    const result = await agent.run('user prompt');
+    await result.evaluations;
+
+    expect(mockEvaluator.evaluate).toHaveBeenCalledWith('user prompt', 'Agent response');
+  });
+
+  it('getConfig() returns the agent config', () => {
+    const agent = new ManagedAgent(agentConfig, mockProvider);
+    expect(agent.getConfig()).toBe(agentConfig);
+  });
+
+  it('getProvider() returns the provider', () => {
+    const agent = new ManagedAgent(agentConfig, mockProvider);
+    expect(agent.getProvider()).toBe(mockProvider);
+  });
+});

packages/sdk/server-ai/src/LDAIClientImpl.ts

@@ -3,6 +3,7 @@ import { randomUUID } from 'node:crypto';
 
 import { LDContext, LDLogger } from '@launchdarkly/js-server-sdk-common';
 
+import { ManagedAgent } from './api/agent/ManagedAgent';
 import { TrackedChat } from './api/chat';
 import {
   LDAIAgentConfig,
@@ -37,6 +38,7 @@ import { aiSdkLanguage, aiSdkName, aiSdkVersion } from './sdkInfo';
 const TRACK_SDK_INFO = '$ld:ai:sdk:info';
 const TRACK_USAGE_COMPLETION_CONFIG = '$ld:ai:usage:completion-config';
 const TRACK_USAGE_CREATE_CHAT = '$ld:ai:usage:create-chat';
+const TRACK_USAGE_CREATE_AGENT = '$ld:ai:usage:create-agent';
 const TRACK_USAGE_JUDGE_CONFIG = '$ld:ai:usage:judge-config';
 const TRACK_USAGE_CREATE_JUDGE = '$ld:ai:usage:create-judge';
 const TRACK_USAGE_AGENT_CONFIG = '$ld:ai:usage:agent-config';
@@ -420,6 +422,44 @@ export class LDAIClientImpl implements LDAIClient {
     return this.createChat(key, context, defaultValue, variables, defaultAiProvider);
   }
 
+  async createAgent(
+    key: string,
+    context: LDContext,
+    defaultValue?: LDAIAgentConfigDefault,
+    variables?: Record<string, unknown>,
+    defaultAiProvider?: SupportedAIProvider,
+  ): Promise<ManagedAgent | undefined> {
+    this._ldClient.track(TRACK_USAGE_CREATE_AGENT, context, key, 1);
+
+    const config = await this._agentConfig(
+      key,
+      context,
+      defaultValue ?? disabledAIConfig,
+      variables,
+    );
+
+    if (!config.enabled) {
+      this._logger?.info(`Agent configuration is disabled: ${key}`);
+      return undefined;
+    }
+
+    const provider = await AIProviderFactory.create(config, this._logger, defaultAiProvider);
+    if (!provider) {
+      return undefined;
+    }
+
+    const evaluator = await this._buildEvaluator(
+      config.judgeConfiguration?.judges ?? [],
+      context,
+      variables,
+      defaultAiProvider,
+    );
+
+    const configWithEvaluator: LDAIAgentConfig = { ...config, evaluator };
+
+    return new ManagedAgent(configWithEvaluator, provider, this._logger);
+  }
+
   /**
    * @deprecated Use `createChat` instead. This method will be removed in a future version.
    */

packages/sdk/server-ai/src/api/LDAIClient.ts

@@ -1,5 +1,6 @@
 import { LDContext } from '@launchdarkly/js-server-sdk-common';
 
+import { ManagedAgent } from './agent/ManagedAgent';
 import { TrackedChat } from './chat';
 import {
   LDAIAgentConfig,
@@ -295,6 +296,25 @@ export interface LDAIClient {
     defaultAiProvider?: SupportedAIProvider,
   ): Promise<TrackedChat | undefined>;
 
+  /**
+   * Creates and returns a new ManagedAgent instance for agent interactions.
+   * Evaluations are wired automatically and exposed on ManagedResult.evaluations.
+   *
+   * @param key The key identifying the agent AI config to use.
+   * @param context The standard LDContext used when evaluating flags.
+   * @param defaultValue Optional fallback when the configuration is not available from LaunchDarkly.
+   * @param variables Dictionary of values for instruction interpolation.
+   * @param defaultAiProvider Optional default AI provider to use.
+   * @returns A promise that resolves to the ManagedAgent instance, or undefined if disabled.
+   */
+  createAgent(
+    key: string,
+    context: LDContext,
+    defaultValue?: LDAIAgentConfigDefault,
+    variables?: Record<string, unknown>,
+    defaultAiProvider?: SupportedAIProvider,
+  ): Promise<ManagedAgent | undefined>;
+
   /**
    * @deprecated Use `createChat` instead. This method will be removed in a future version.
    */

packages/sdk/server-ai/src/api/agent/ManagedAgent.ts

@@ -0,0 +1,93 @@
+import { LDLogger } from '@launchdarkly/js-server-sdk-common';
+
+import { ChatResponse } from '../chat/types';
+import { LDAIAgentConfig } from '../config/types';
+import { LDJudgeResult } from '../judge/types';
+import { LDAIMetricSummary, ManagedResult } from '../model/types';
+import { AIProvider } from '../providers/AIProvider';
+
+/**
+ * ManagedAgent provides agent invocation with automatic judge evaluation.
+ *
+ * This is the agent-mode analogue of TrackedChat (ManagedModel). It invokes
+ * the provider, tracks metrics, and wires judge evaluations into a single
+ * Promise exposed on ManagedResult.evaluations.
+ *
+ * Obtain an instance via `LDAIClient.createAgent()`.
+ */
+export class ManagedAgent {
+  constructor(
+    protected readonly aiAgentConfig: LDAIAgentConfig,
+    protected readonly provider: AIProvider,
+    private readonly _logger?: LDLogger,
+  ) {}
+
+  /**
+   * Invoke the agent with a prompt string and return a ManagedResult.
+   *
+   * run() returns before ManagedResult.evaluations resolves. Awaiting evaluations
+   * guarantees both evaluation and tracker.trackJudgeResult() are complete.
+   *
+   * @param prompt The user input to send to the agent.
+   * @returns Promise resolving to ManagedResult (before evaluations settle).
+   */
+  async run(prompt: string): Promise<ManagedResult> {
+    const tracker = this.aiAgentConfig.createTracker!();
+
+    const userMessage = { role: 'user' as const, content: prompt };
+    const allMessages = [userMessage];
+
+    // Delegate to provider-specific implementation with tracking
+    const response = await tracker.trackMetricsOf(
+      (result: ChatResponse) => result.metrics,
+      () => this.provider.invokeModel(allMessages),
+    );
+
+    // Build the metric summary from response metrics + resumption token
+    const metrics: LDAIMetricSummary = {
+      success: response.metrics.success,
+      usage: response.metrics.usage,
+      toolCalls: response.metrics.toolCalls,
+      durationMs: response.metrics.durationMs,
+      resumptionToken: tracker.resumptionToken,
+    };
+
+    const output = response.message.content;
+
+    // Wire evaluation + tracking into a single Promise.
+    // run() returns before this resolves — awaiting evaluations guarantees
+    // both evaluation and tracking are complete.
+    const evaluator = this.aiAgentConfig.evaluator;
+    let evaluations: Promise<LDJudgeResult[]>;
+    if (evaluator && evaluator.judgeConfiguration.judges.length > 0) {
+      evaluations = evaluator.evaluate(prompt, output).then((results) => {
+        results.forEach((judgeResult) => {
+          tracker.trackJudgeResult(judgeResult);
+        });
+        return results;
+      });
+    } else {
+      evaluations = Promise.resolve([]);
+    }
+
+    return {
+      content: output,
+      metrics,
+      evaluations,
+    };
+  }
+
+  /**
+   * Get the underlying AI agent configuration.
+   */
+  getConfig(): LDAIAgentConfig {
+    return this.aiAgentConfig;
+  }
+
+  /**
+   * Get the underlying AI provider instance.
+   */
+  getProvider(): AIProvider {
+    return this.provider;
+  }
+}

packages/sdk/server-ai/src/api/agent/index.ts

@@ -0,0 +1 @@
+export { ManagedAgent } from './ManagedAgent';

packages/sdk/server-ai/src/api/index.ts

@@ -1,4 +1,5 @@
 export * from './config';
+export * from './agent';
 export * from './chat';
 export * from './graph';
 export * from './judge';