diff --git a/packages/sdk/server-ai/__tests__/ManagedAgent.test.ts b/packages/sdk/server-ai/__tests__/ManagedAgent.test.ts new file mode 100644 index 0000000000..98a2a642d0 --- /dev/null +++ b/packages/sdk/server-ai/__tests__/ManagedAgent.test.ts @@ -0,0 +1,170 @@ +import { ManagedAgent } from '../src/api/agent/ManagedAgent'; +import { LDAIConfigTracker } from '../src/api/config/LDAIConfigTracker'; +import { LDAIAgentConfig } from '../src/api/config/types'; +import { Evaluator } from '../src/api/judge/Evaluator'; +import { LDJudgeResult } from '../src/api/judge/types'; +import { RunnerResult } from '../src/api/model/types'; +import { Runner } from '../src/api/providers/Runner'; + +describe('ManagedAgent', () => { + let mockRunner: jest.Mocked; + let mockTracker: jest.Mocked; + let agentConfig: LDAIAgentConfig; + + const runnerResult: RunnerResult = { + content: 'Agent response', + metrics: { success: true }, + }; + + beforeEach(() => { + mockRunner = { + run: jest.fn().mockResolvedValue(runnerResult), + }; + + mockTracker = { + trackMetricsOf: jest.fn().mockImplementation(async (_extractor: any, func: any) => func()), + trackJudgeResult: jest.fn(), + resumptionToken: 'agent-resumption-token', + getTrackData: jest.fn().mockReturnValue({}), + trackDuration: jest.fn(), + trackTokens: jest.fn(), + trackSuccess: jest.fn(), + trackError: jest.fn(), + trackFeedback: jest.fn(), + trackTimeToFirstToken: jest.fn(), + trackDurationOf: jest.fn(), + trackOpenAIMetrics: jest.fn(), + trackBedrockConverseMetrics: jest.fn(), + trackVercelAIMetrics: jest.fn(), + getSummary: jest.fn(), + } as any; + + agentConfig = { + key: 'test-agent', + enabled: true, + instructions: 'You are a helpful agent.', + model: { name: 'gpt-4' }, + provider: { name: 'openai' }, + createTracker: () => mockTracker, + }; + }); + + it('returns a ManagedResult with content and metrics', async () => { + const agent = new ManagedAgent(agentConfig, mockRunner); + const result = await agent.run('Hello agent'); + + expect(result.content).toBe('Agent response'); + expect(result.metrics.success).toBe(true); + expect(result.metrics.resumptionToken).toBe('agent-resumption-token'); + }); + + it('passes the prompt directly to the runner', async () => { + const agent = new ManagedAgent(agentConfig, mockRunner); + await agent.run('My question'); + + expect(mockRunner.run).toHaveBeenCalledWith('My question'); + }); + + it('resolves to empty evaluations when no evaluator configured', async () => { + const agent = new ManagedAgent(agentConfig, mockRunner); + const result = await agent.run('Hello'); + const evaluations = await result.evaluations; + expect(evaluations).toEqual([]); + }); + + it('resolves to empty evaluations with noop evaluator', async () => { + const configWithNoop: LDAIAgentConfig = { + ...agentConfig, + evaluator: Evaluator.noop(), + }; + const agent = new ManagedAgent(configWithNoop, mockRunner); + const result = await agent.run('Hello'); + const evaluations = await result.evaluations; + expect(evaluations).toEqual([]); + }); + + it('awaiting evaluations calls tracker.trackJudgeResult', async () => { + const judgeResult: LDJudgeResult = { + success: true, + sampled: true, + score: 0.85, + metricKey: 'quality', + }; + const mockEvaluator = { + judgeConfiguration: { judges: [{ key: 'judge-1', samplingRate: 1.0 }] }, + evaluate: jest.fn().mockResolvedValue([judgeResult]), + judges: new Map(), + } as unknown as Evaluator; + + const configWithEvaluator: LDAIAgentConfig = { + ...agentConfig, + evaluator: mockEvaluator, + }; + + const agent = new ManagedAgent(configWithEvaluator, mockRunner); + const result = await agent.run('Hello'); + + await result.evaluations; + expect(mockTracker.trackJudgeResult).toHaveBeenCalledWith(judgeResult); + }); + + it('passes the prompt to evaluator.evaluate as input', async () => { + const mockEvaluator = { + judgeConfiguration: { judges: [{ key: 'judge-1', samplingRate: 1.0 }] }, + evaluate: jest.fn().mockResolvedValue([]), + judges: new Map(), + } as unknown as Evaluator; + + const configWithEvaluator: LDAIAgentConfig = { + ...agentConfig, + evaluator: mockEvaluator, + }; + + const agent = new ManagedAgent(configWithEvaluator, mockRunner); + const result = await agent.run('user prompt'); + await result.evaluations; + + expect(mockEvaluator.evaluate).toHaveBeenCalledWith('user prompt', 'Agent response'); + }); + + it('returns before evaluations resolve', async () => { + let resolveEval!: (v: LDJudgeResult[]) => void; + const slowEvaluator = { + judgeConfiguration: { judges: [{ key: 'judge-1', samplingRate: 1.0 }] }, + evaluate: jest.fn().mockReturnValue( + new Promise((resolve) => { + resolveEval = resolve; + }), + ), + judges: new Map(), + } as unknown as Evaluator; + + const configWithEvaluator: LDAIAgentConfig = { + ...agentConfig, + evaluator: slowEvaluator, + }; + + const agent = new ManagedAgent(configWithEvaluator, mockRunner); + + let evaluationsResolved = false; + const result = await agent.run('Hello'); + + expect(result.content).toBe('Agent response'); + + result.evaluations.then(() => { + evaluationsResolved = true; + }); + + await Promise.resolve(); + expect(evaluationsResolved).toBe(false); + + resolveEval([{ success: true, sampled: true, score: 0.9 }]); + await result.evaluations; + expect(evaluationsResolved).toBe(true); + }); + + it('exposes the agent config via getConfig', () => { + const agent = new ManagedAgent(agentConfig, mockRunner); + expect(agent.getConfig()).toBe(agentConfig); + }); +}); diff --git a/packages/sdk/server-ai/src/LDAIClientImpl.ts b/packages/sdk/server-ai/src/LDAIClientImpl.ts index 9cd9dbf6e5..0ba86adc4f 100644 --- a/packages/sdk/server-ai/src/LDAIClientImpl.ts +++ b/packages/sdk/server-ai/src/LDAIClientImpl.ts @@ -3,6 +3,7 @@ import { randomUUID } from 'node:crypto'; import { LDContext, LDLogger } from '@launchdarkly/js-server-sdk-common'; +import { ManagedAgent } from './api/agent/ManagedAgent'; import { ManagedModel } from './api/chat'; import { LDAIAgentConfig, @@ -38,6 +39,7 @@ import { aiSdkLanguage, aiSdkName, aiSdkVersion } from './sdkInfo'; const TRACK_SDK_INFO = '$ld:ai:sdk:info'; const TRACK_USAGE_COMPLETION_CONFIG = '$ld:ai:usage:completion-config'; const TRACK_USAGE_CREATE_CHAT = '$ld:ai:usage:create-chat'; +const TRACK_USAGE_CREATE_AGENT = '$ld:ai:usage:create-agent'; const TRACK_USAGE_JUDGE_CONFIG = '$ld:ai:usage:judge-config'; const TRACK_USAGE_CREATE_JUDGE = '$ld:ai:usage:create-judge'; const TRACK_USAGE_AGENT_CONFIG = '$ld:ai:usage:agent-config'; @@ -72,6 +74,30 @@ function runnerFromAIProvider(provider: AIProvider, config: LDAICompletionConfig }; } +/** + * Adapt a (deprecated) AIProvider to the Runner protocol for agent configs. + * Prepends the agent's instructions as a system message so existing + * AIProvider-based agent flows preserve their instruction behavior under the + * stateless Runner contract. + */ +function runnerFromAIProviderForAgent(provider: AIProvider, config: LDAIAgentConfig): Runner { + return { + async run(input: string): Promise { + const messages: LDMessage[] = []; + if (config.instructions) { + messages.push({ role: 'system', content: config.instructions }); + } + messages.push({ role: 'user', content: input }); + const response = await provider.invokeModel(messages); + return { + content: response.message.content, + metrics: response.metrics, + raw: response, + }; + }, + }; +} + export class LDAIClientImpl implements LDAIClient { private _logger?: LDLogger; @@ -439,6 +465,45 @@ export class LDAIClientImpl implements LDAIClient { return new ManagedModel(configWithEvaluator, runner, this._logger); } + async createAgent( + key: string, + context: LDContext, + defaultValue?: LDAIAgentConfigDefault, + variables?: Record, + defaultAiProvider?: SupportedAIProvider, + ): Promise { + this._ldClient.track(TRACK_USAGE_CREATE_AGENT, context, key, 1); + + const config = await this._agentConfig( + key, + context, + defaultValue ?? disabledAIConfig, + variables, + ); + + if (!config.enabled) { + this._logger?.info(`Agent configuration is disabled: ${key}`); + return undefined; + } + + const provider = await AIProviderFactory.create(config, this._logger, defaultAiProvider); + if (!provider) { + return undefined; + } + + const evaluator = await this._buildEvaluator( + config.judgeConfiguration?.judges ?? [], + context, + variables, + defaultAiProvider, + ); + + const configWithEvaluator: LDAIAgentConfig = { ...config, evaluator }; + + const runner = runnerFromAIProviderForAgent(provider, configWithEvaluator); + return new ManagedAgent(configWithEvaluator, runner, this._logger); + } + /** * @deprecated Use `createModel` instead. This method will be removed in a future version. */ diff --git a/packages/sdk/server-ai/src/api/LDAIClient.ts b/packages/sdk/server-ai/src/api/LDAIClient.ts index 02dfb43e3d..fe80c0bc31 100644 --- a/packages/sdk/server-ai/src/api/LDAIClient.ts +++ b/packages/sdk/server-ai/src/api/LDAIClient.ts @@ -1,5 +1,6 @@ import { LDContext } from '@launchdarkly/js-server-sdk-common'; +import { ManagedAgent } from './agent/ManagedAgent'; import { ManagedModel } from './chat'; import { LDAIAgentConfig, @@ -275,6 +276,25 @@ export interface LDAIClient { defaultAiProvider?: SupportedAIProvider, ): Promise; + /** + * Creates and returns a new ManagedAgent instance for agent interactions. + * Evaluations are wired automatically and exposed on ManagedResult.evaluations. + * + * @param key The key identifying the agent AI config to use. + * @param context The standard LDContext used when evaluating flags. + * @param defaultValue Optional fallback when the configuration is not available from LaunchDarkly. + * @param variables Dictionary of values for instruction interpolation. + * @param defaultAiProvider Optional default AI provider to use. + * @returns A promise that resolves to the ManagedAgent instance, or undefined if disabled. + */ + createAgent( + key: string, + context: LDContext, + defaultValue?: LDAIAgentConfigDefault, + variables?: Record, + defaultAiProvider?: SupportedAIProvider, + ): Promise; + /** * @deprecated Use `createModel` instead. This method will be removed in a future version. */ diff --git a/packages/sdk/server-ai/src/api/agent/ManagedAgent.ts b/packages/sdk/server-ai/src/api/agent/ManagedAgent.ts new file mode 100644 index 0000000000..aa1cd8d604 --- /dev/null +++ b/packages/sdk/server-ai/src/api/agent/ManagedAgent.ts @@ -0,0 +1,80 @@ +import { LDLogger } from '@launchdarkly/js-server-sdk-common'; + +import { LDAIAgentConfig } from '../config/types'; +import { LDJudgeResult } from '../judge/types'; +import { LDAIMetricSummary, ManagedResult, RunnerResult } from '../model/types'; +import { Runner } from '../providers/Runner'; + +/** + * ManagedAgent provides agent invocation with automatic tracking and automatic + * judge evaluation. + * + * The class is stateless: each `run()` call sends the prompt directly to the + * underlying `Runner` and returns a `ManagedResult`. Conversation history, + * if any, must be managed by the caller (or by the Runner implementation). + * + * Obtain an instance via `LDAIClient.createAgent()`. + */ +export class ManagedAgent { + constructor( + protected readonly aiAgentConfig: LDAIAgentConfig, + protected readonly runner: Runner, + private readonly _logger?: LDLogger, + ) {} + + /** + * Invoke the agent with a prompt string and return a ManagedResult. + * + * `run()` resolves before `ManagedResult.evaluations` resolves. Awaiting + * `evaluations` guarantees both judge evaluation and tracker.trackJudgeResult() + * are complete. + * + * @param prompt The user input to send to the agent. + * @returns Promise resolving to ManagedResult (before evaluations settle). + */ + async run(prompt: string): Promise { + const tracker = this.aiAgentConfig.createTracker!(); + + const result = await tracker.trackMetricsOf( + (r: RunnerResult) => r.metrics, + () => this.runner.run(prompt), + ); + + const metrics: LDAIMetricSummary = { + success: result.metrics.success, + usage: result.metrics.usage, + toolCalls: result.metrics.toolCalls, + durationMs: result.metrics.durationMs, + resumptionToken: tracker.resumptionToken, + }; + + const output = result.content; + const evaluator = this.aiAgentConfig.evaluator; + let evaluations: Promise; + if (evaluator) { + evaluations = evaluator.evaluate(prompt, output).then((results) => { + results.forEach((judgeResult) => { + tracker.trackJudgeResult(judgeResult); + }); + return results; + }); + } else { + evaluations = Promise.resolve([]); + } + + return { + content: output, + metrics, + raw: result.raw, + parsed: result.parsed, + evaluations, + }; + } + + /** + * Get the underlying AI agent configuration used to initialize this ManagedAgent. + */ + getConfig(): LDAIAgentConfig { + return this.aiAgentConfig; + } +} diff --git a/packages/sdk/server-ai/src/api/agent/index.ts b/packages/sdk/server-ai/src/api/agent/index.ts new file mode 100644 index 0000000000..68bcb93777 --- /dev/null +++ b/packages/sdk/server-ai/src/api/agent/index.ts @@ -0,0 +1 @@ +export { ManagedAgent } from './ManagedAgent'; diff --git a/packages/sdk/server-ai/src/api/index.ts b/packages/sdk/server-ai/src/api/index.ts index 53801ec6d1..a33e35d977 100644 --- a/packages/sdk/server-ai/src/api/index.ts +++ b/packages/sdk/server-ai/src/api/index.ts @@ -1,4 +1,5 @@ export * from './config'; +export * from './agent'; export * from './chat'; export * from './graph'; export * from './judge';