diff --git a/packages/sdk/server-ai/__tests__/ManagedModel.test.ts b/packages/sdk/server-ai/__tests__/ManagedModel.test.ts new file mode 100644 index 0000000000..64f16ea15c --- /dev/null +++ b/packages/sdk/server-ai/__tests__/ManagedModel.test.ts @@ -0,0 +1,130 @@ +import { ManagedModel } from '../src/api/chat/ManagedModel'; +import { LDAIConfigTracker } from '../src/api/config/LDAIConfigTracker'; +import { LDAICompletionConfig } from '../src/api/config/types'; +import { RunnerResult } from '../src/api/model/types'; +import { Runner } from '../src/api/providers/Runner'; + +describe('ManagedModel', () => { + let mockRunner: jest.Mocked; + let mockTracker: jest.Mocked; + let aiConfig: LDAICompletionConfig; + + beforeEach(() => { + mockRunner = { + run: jest.fn(), + }; + + mockTracker = { + trackMetricsOf: jest.fn(), + trackDuration: jest.fn(), + trackTokens: jest.fn(), + trackSuccess: jest.fn(), + trackError: jest.fn(), + trackFeedback: jest.fn(), + trackTimeToFirstToken: jest.fn(), + trackDurationOf: jest.fn(), + trackOpenAIMetrics: jest.fn(), + trackBedrockConverseMetrics: jest.fn(), + trackVercelAIMetrics: jest.fn(), + getSummary: jest.fn(), + resumptionToken: 'resumption-token-123', + } as any; + + aiConfig = { + key: 'test-config', + enabled: true, + messages: [{ role: 'system', content: 'You are a helpful assistant.' }], + model: { name: 'gpt-4' }, + provider: { name: 'openai' }, + createTracker: () => mockTracker, + }; + }); + + it('passes the prompt directly to the runner without prepending config messages', async () => { + const runnerResult: RunnerResult = { + content: 'Response from model', + metrics: { success: true, usage: { total: 10, input: 4, output: 6 } }, + }; + + mockTracker.trackMetricsOf.mockImplementation(async (_extractor, func) => func()); + mockRunner.run.mockResolvedValue(runnerResult); + + const model = new ManagedModel(aiConfig, mockRunner); + await model.run('Hello'); + + expect(mockRunner.run).toHaveBeenCalledTimes(1); + expect(mockRunner.run).toHaveBeenCalledWith('Hello'); + }); + + it('returns a ManagedResult with content, metrics, and an evaluations promise', async () => { + const runnerResult: RunnerResult = { + content: 'Hi there', + metrics: { + success: true, + usage: { total: 12, input: 5, output: 7 }, + toolCalls: ['tool-1'], + durationMs: 42, + }, + raw: { providerSpecific: true }, + }; + + mockTracker.trackMetricsOf.mockImplementation(async (_extractor, func) => func()); + mockRunner.run.mockResolvedValue(runnerResult); + + const model = new ManagedModel(aiConfig, mockRunner); + const result = await model.run('say hi'); + + expect(result.content).toBe('Hi there'); + expect(result.metrics).toEqual({ + success: true, + usage: { total: 12, input: 5, output: 7 }, + toolCalls: ['tool-1'], + durationMs: 42, + resumptionToken: 'resumption-token-123', + }); + expect(result.raw).toEqual({ providerSpecific: true }); + await expect(result.evaluations).resolves.toEqual([]); + }); + + it('forwards the runner result through tracker.trackMetricsOf', async () => { + const runnerResult: RunnerResult = { + content: 'tracked', + metrics: { success: true, usage: { total: 1, input: 1, output: 0 } }, + }; + + mockTracker.trackMetricsOf.mockImplementation(async (_extractor, func) => func()); + mockRunner.run.mockResolvedValue(runnerResult); + + const model = new ManagedModel(aiConfig, mockRunner); + await model.run('prompt'); + + expect(mockTracker.trackMetricsOf).toHaveBeenCalledTimes(1); + const [extractor] = mockTracker.trackMetricsOf.mock.calls[0]; + // The extractor should pull metrics off the RunnerResult + expect(extractor(runnerResult)).toBe(runnerResult.metrics); + }); + + it('does not retain conversation state across runs', async () => { + const runnerResult: RunnerResult = { + content: 'ok', + metrics: { success: true, usage: { total: 1, input: 1, output: 0 } }, + }; + + mockTracker.trackMetricsOf.mockImplementation(async (_extractor, func) => func()); + mockRunner.run.mockResolvedValue(runnerResult); + + const model = new ManagedModel(aiConfig, mockRunner); + + await model.run('first'); + await model.run('second'); + + // Each call passes only the latest prompt — no accumulated history. + expect(mockRunner.run).toHaveBeenNthCalledWith(1, 'first'); + expect(mockRunner.run).toHaveBeenNthCalledWith(2, 'second'); + }); + + it('exposes the AI config via getConfig', () => { + const model = new ManagedModel(aiConfig, mockRunner); + expect(model.getConfig()).toBe(aiConfig); + }); +}); diff --git a/packages/sdk/server-ai/__tests__/TrackedChat.test.ts b/packages/sdk/server-ai/__tests__/TrackedChat.test.ts deleted file mode 100644 index 75681b0f83..0000000000 --- a/packages/sdk/server-ai/__tests__/TrackedChat.test.ts +++ /dev/null @@ -1,231 +0,0 @@ -import { TrackedChat } from '../src/api/chat/TrackedChat'; -import { ChatResponse } from '../src/api/chat/types'; -import { LDAIConfigTracker } from '../src/api/config/LDAIConfigTracker'; -import { LDAICompletionConfig, LDMessage } from '../src/api/config/types'; -import { AIProvider } from '../src/api/providers/AIProvider'; - -describe('TrackedChat', () => { - let mockProvider: jest.Mocked; - let mockTracker: jest.Mocked; - let aiConfig: LDAICompletionConfig; - - beforeEach(() => { - // Mock the AIProvider - mockProvider = { - invokeModel: jest.fn(), - } as any; - - // Mock the LDAIConfigTracker - mockTracker = { - trackMetricsOf: jest.fn(), - trackDuration: jest.fn(), - trackTokens: jest.fn(), - trackSuccess: jest.fn(), - trackError: jest.fn(), - trackFeedback: jest.fn(), - trackTimeToFirstToken: jest.fn(), - trackDurationOf: jest.fn(), - trackOpenAIMetrics: jest.fn(), - trackBedrockConverseMetrics: jest.fn(), - trackVercelAIMetrics: jest.fn(), - getSummary: jest.fn(), - } as any; - - // Create a basic AI config - aiConfig = { - key: 'test-config', - enabled: true, - messages: [{ role: 'system', content: 'You are a helpful assistant.' }], - model: { name: 'gpt-4' }, - provider: { name: 'openai' }, - createTracker: () => mockTracker, - }; - }); - - describe('appendMessages', () => { - it('appends messages to the conversation history', () => { - const chat = new TrackedChat(aiConfig, mockProvider); - - const messagesToAppend: LDMessage[] = [ - { role: 'user', content: 'Hello' }, - { role: 'assistant', content: 'Hi there!' }, - ]; - - chat.appendMessages(messagesToAppend); - - const messages = chat.getMessages(false); - expect(messages).toHaveLength(2); - expect(messages[0]).toEqual({ role: 'user', content: 'Hello' }); - expect(messages[1]).toEqual({ role: 'assistant', content: 'Hi there!' }); - }); - - it('appends multiple message batches sequentially', () => { - const chat = new TrackedChat(aiConfig, mockProvider); - - chat.appendMessages([{ role: 'user', content: 'First message' }]); - chat.appendMessages([{ role: 'assistant', content: 'Second message' }]); - chat.appendMessages([{ role: 'user', content: 'Third message' }]); - - const messages = chat.getMessages(false); - expect(messages).toHaveLength(3); - expect(messages[0].content).toBe('First message'); - expect(messages[1].content).toBe('Second message'); - expect(messages[2].content).toBe('Third message'); - }); - - it('handles empty message array', () => { - const chat = new TrackedChat(aiConfig, mockProvider); - - chat.appendMessages([]); - - const messages = chat.getMessages(false); - expect(messages).toHaveLength(0); - }); - }); - - describe('getMessages', () => { - it('returns only conversation history when includeConfigMessages is false', () => { - const chat = new TrackedChat(aiConfig, mockProvider); - - chat.appendMessages([ - { role: 'user', content: 'User message' }, - { role: 'assistant', content: 'Assistant message' }, - ]); - - const messages = chat.getMessages(false); - - expect(messages).toHaveLength(2); - expect(messages[0]).toEqual({ role: 'user', content: 'User message' }); - expect(messages[1]).toEqual({ role: 'assistant', content: 'Assistant message' }); - }); - - it('returns only conversation history when includeConfigMessages is omitted (defaults to false)', () => { - const chat = new TrackedChat(aiConfig, mockProvider); - - chat.appendMessages([{ role: 'user', content: 'User message' }]); - - const messages = chat.getMessages(); - - expect(messages).toHaveLength(1); - expect(messages[0]).toEqual({ role: 'user', content: 'User message' }); - }); - - it('returns config messages prepended when includeConfigMessages is true', () => { - const chat = new TrackedChat(aiConfig, mockProvider); - - chat.appendMessages([ - { role: 'user', content: 'User message' }, - { role: 'assistant', content: 'Assistant message' }, - ]); - - const messages = chat.getMessages(true); - - expect(messages).toHaveLength(3); - expect(messages[0]).toEqual({ role: 'system', content: 'You are a helpful assistant.' }); - expect(messages[1]).toEqual({ role: 'user', content: 'User message' }); - expect(messages[2]).toEqual({ role: 'assistant', content: 'Assistant message' }); - }); - - it('returns only config messages when no conversation history exists and includeConfigMessages is true', () => { - const chat = new TrackedChat(aiConfig, mockProvider); - - const messages = chat.getMessages(true); - - expect(messages).toHaveLength(1); - expect(messages[0]).toEqual({ role: 'system', content: 'You are a helpful assistant.' }); - }); - - it('returns empty array when no messages exist and includeConfigMessages is false', () => { - const configWithoutMessages: LDAICompletionConfig = { - ...aiConfig, - messages: [], - }; - const chat = new TrackedChat(configWithoutMessages, mockProvider); - - const messages = chat.getMessages(false); - - expect(messages).toHaveLength(0); - }); - - it('returns a copy of the messages array (not a reference)', () => { - const chat = new TrackedChat(aiConfig, mockProvider); - - chat.appendMessages([{ role: 'user', content: 'Original message' }]); - - const messages1 = chat.getMessages(); - const messages2 = chat.getMessages(); - - expect(messages1).not.toBe(messages2); - expect(messages1).toEqual(messages2); - - // Modifying returned array should not affect internal state - messages1.push({ role: 'assistant', content: 'Modified' }); - - const messages3 = chat.getMessages(); - expect(messages3).toHaveLength(1); - expect(messages3[0].content).toBe('Original message'); - }); - - it('handles undefined config messages gracefully', () => { - const configWithoutMessages: LDAICompletionConfig = { - ...aiConfig, - messages: undefined, - }; - const chat = new TrackedChat(configWithoutMessages, mockProvider); - - chat.appendMessages([{ role: 'user', content: 'User message' }]); - - const messagesWithConfig = chat.getMessages(true); - expect(messagesWithConfig).toHaveLength(1); - expect(messagesWithConfig[0].content).toBe('User message'); - - const messagesWithoutConfig = chat.getMessages(false); - expect(messagesWithoutConfig).toHaveLength(1); - expect(messagesWithoutConfig[0].content).toBe('User message'); - }); - }); - - describe('integration with invoke', () => { - it('adds messages from invoke to history accessible via getMessages', async () => { - const mockResponse: ChatResponse = { - message: { role: 'assistant', content: 'Response from model' }, - metrics: { success: true }, - }; - - mockTracker.trackMetricsOf.mockImplementation(async (extractor, func) => func()); - - mockProvider.invokeModel.mockResolvedValue(mockResponse); - - const chat = new TrackedChat(aiConfig, mockProvider); - - await chat.invoke('Hello'); - - const messages = chat.getMessages(false); - expect(messages).toHaveLength(2); - expect(messages[0]).toEqual({ role: 'user', content: 'Hello' }); - expect(messages[1]).toEqual({ role: 'assistant', content: 'Response from model' }); - }); - - it('preserves appended messages when invoking', async () => { - const mockResponse: ChatResponse = { - message: { role: 'assistant', content: 'Response' }, - metrics: { success: true }, - }; - - mockTracker.trackMetricsOf.mockImplementation(async (extractor, func) => func()); - - mockProvider.invokeModel.mockResolvedValue(mockResponse); - - const chat = new TrackedChat(aiConfig, mockProvider); - - chat.appendMessages([{ role: 'user', content: 'Pre-appended message' }]); - await chat.invoke('New user input'); - - const messages = chat.getMessages(false); - expect(messages).toHaveLength(3); - expect(messages[0].content).toBe('Pre-appended message'); - expect(messages[1].content).toBe('New user input'); - expect(messages[2].content).toBe('Response'); - }); - }); -}); diff --git a/packages/sdk/server-ai/examples/chat-judge/src/index.ts b/packages/sdk/server-ai/examples/chat-judge/src/index.ts index 9145081cf6..e74877aa48 100644 --- a/packages/sdk/server-ai/examples/chat-judge/src/index.ts +++ b/packages/sdk/server-ai/examples/chat-judge/src/index.ts @@ -43,11 +43,11 @@ async function main() { enabled: false, }; - const chat = await aiClient.createChat(aiConfigKey, context, defaultValue, { + const model = await aiClient.createModel(aiConfigKey, context, defaultValue, { companyName: 'LaunchDarkly', }); - if (!chat) { + if (!model) { console.log('*** AI chat configuration is not enabled'); process.exit(0); } @@ -56,15 +56,14 @@ async function main() { const userInput = 'How can LaunchDarkly help me?'; console.log('User Input:', userInput); - // The invoke method will automatically evaluate the chat response with any judges defined - // in the AI config. - const chatResponse = await chat.invoke(userInput); - console.log('Chat Response:', chatResponse.message.content); + // The run() method invokes the model and returns a ManagedResult. + const result = await model.run(userInput); + console.log('Chat Response:', result.content); // Judge evaluations run asynchronously and do not block your application. // Results are automatically sent to LaunchDarkly for AI config metrics. // You only need to await if you want to access the evaluation results in your code. - const evalResults = await chatResponse.evaluations; + const evalResults = await result.evaluations; console.log('Judge results:', JSON.stringify(evalResults, null, 2)); console.log('Success.'); diff --git a/packages/sdk/server-ai/examples/chat-observability/src/index.ts b/packages/sdk/server-ai/examples/chat-observability/src/index.ts index d7f5818597..b4f25e703b 100644 --- a/packages/sdk/server-ai/examples/chat-observability/src/index.ts +++ b/packages/sdk/server-ai/examples/chat-observability/src/index.ts @@ -47,12 +47,12 @@ async function main() { // provider: { name: 'openai' }, // messages: [...] // }; - // const chat = await aiClient.createChat(aiConfigKey, context, defaultValue, { example_type: 'observability_demo' }); - const chat = await aiClient.createChat(aiConfigKey, context, undefined, { + // const model = await aiClient.createModel(aiConfigKey, context, defaultValue, { example_type: 'observability_demo' }); + const model = await aiClient.createModel(aiConfigKey, context, undefined, { example_type: 'observability_demo', }); - if (!chat) { + if (!model) { console.log('*** AI chat configuration is not enabled'); ldClient.close(); process.exit(0); @@ -62,8 +62,8 @@ async function main() { const userInput = 'What is feature flagging in 2 sentences?'; console.log('User Input:', userInput); - const response = await chat.invoke(userInput); - console.log('Chat Response:', response.message.content); + const result = await model.run(userInput); + console.log('Chat Response:', result.content); console.log('\nSuccess.'); } catch (err) { diff --git a/packages/sdk/server-ai/examples/tracked-chat/src/index.ts b/packages/sdk/server-ai/examples/tracked-chat/src/index.ts index fd350e068c..f3b200ea2d 100644 --- a/packages/sdk/server-ai/examples/tracked-chat/src/index.ts +++ b/packages/sdk/server-ai/examples/tracked-chat/src/index.ts @@ -46,12 +46,12 @@ async function main() { // provider: { name: 'openai' }, // messages: [...] // }; - // const chat = await aiClient.createChat(aiConfigKey, context, defaultValue, { companyName: 'LaunchDarkly' }); - const chat = await aiClient.createChat(aiConfigKey, context, undefined, { + // const model = await aiClient.createModel(aiConfigKey, context, defaultValue, { companyName: 'LaunchDarkly' }); + const model = await aiClient.createModel(aiConfigKey, context, undefined, { companyName: 'LaunchDarkly', }); - if (!chat) { + if (!model) { console.log('*** AI chat configuration is not enabled'); process.exit(0); } @@ -62,9 +62,9 @@ async function main() { const userInput = 'Hello! Can you help me understand how your company can help me?'; console.log('User Input:', userInput); - const response = await chat.invoke(userInput); + const result = await model.run(userInput); - console.log('AI Response:', response.message.content); + console.log('AI Response:', result.content); console.log('Success.'); } catch (err) { diff --git a/packages/sdk/server-ai/src/LDAIClientImpl.ts b/packages/sdk/server-ai/src/LDAIClientImpl.ts index 983ae1f781..9cd9dbf6e5 100644 --- a/packages/sdk/server-ai/src/LDAIClientImpl.ts +++ b/packages/sdk/server-ai/src/LDAIClientImpl.ts @@ -3,7 +3,7 @@ import { randomUUID } from 'node:crypto'; import { LDContext, LDLogger } from '@launchdarkly/js-server-sdk-common'; -import { TrackedChat } from './api/chat'; +import { ManagedModel } from './api/chat'; import { LDAIAgentConfig, LDAIAgentConfigDefault, @@ -25,7 +25,8 @@ import { AgentGraphDefinition, LDAgentGraphFlagValue, LDGraphTracker } from './a import { Evaluator } from './api/judge/Evaluator'; import { Judge } from './api/judge/Judge'; import { LDAIClient } from './api/LDAIClient'; -import { AIProviderFactory, SupportedAIProvider } from './api/providers'; +import { RunnerResult } from './api/model/types'; +import { AIProvider, AIProviderFactory, Runner, SupportedAIProvider } from './api/providers'; import { LDAIConfigTrackerImpl } from './LDAIConfigTrackerImpl'; import { LDClientMin } from './LDClientMin'; import { LDGraphTrackerImpl } from './LDGraphTrackerImpl'; @@ -51,6 +52,26 @@ const INIT_TRACK_CONTEXT: LDContext = { const disabledAIConfig: LDAIConfigDefault = { enabled: false }; +/** + * Adapt a (deprecated) AIProvider to the Runner protocol. + * Prepends the AIConfig's configured messages to the user prompt so existing + * AIProvider-based flows preserve their system-prompt behavior under the + * stateless Runner contract. + */ +function runnerFromAIProvider(provider: AIProvider, config: LDAICompletionConfig): Runner { + return { + async run(input: string): Promise { + const messages: LDMessage[] = [...(config.messages ?? []), { role: 'user', content: input }]; + const response = await provider.invokeModel(messages); + return { + content: response.message.content, + metrics: response.metrics, + raw: response, + }; + }, + }; +} + export class LDAIClientImpl implements LDAIClient { private _logger?: LDLogger; @@ -314,48 +335,17 @@ export class LDAIClientImpl implements LDAIClient { return this.agentConfigs(agentConfigs, context); } + /** + * @deprecated Use `createModel` instead. This method will be removed in a future version. + */ async createChat( key: string, context: LDContext, defaultValue?: LDAICompletionConfigDefault, variables?: Record, defaultAiProvider?: SupportedAIProvider, - ): Promise { - this._ldClient.track(TRACK_USAGE_CREATE_CHAT, context, key, 1); - const config = await this._completionConfig( - key, - context, - defaultValue ?? disabledAIConfig, - variables, - ); - - if (!config.enabled) { - this._logger?.info(`Chat configuration is disabled: ${key}`); - return undefined; - } - - const provider = await AIProviderFactory.create(config, this._logger, defaultAiProvider); - if (!provider) { - return undefined; - } - - const evaluator = await this._buildEvaluator( - config.judgeConfiguration?.judges ?? [], - context, - variables, - defaultAiProvider, - ); - - // Attach the evaluator to the config for use by the managed layer - const configWithEvaluator: LDAICompletionConfig = { ...config, evaluator }; - - // Build the legacy judges record for TrackedChat backward compat - const judges: Record = {}; - evaluator.judges.forEach((judge, k) => { - judges[k] = judge; - }); - - return new TrackedChat(configWithEvaluator, provider, judges, this._logger); + ): Promise { + return this.createModel(key, context, defaultValue, variables, defaultAiProvider); } async createJudge( @@ -410,8 +400,47 @@ export class LDAIClientImpl implements LDAIClient { } } + async createModel( + key: string, + context: LDContext, + defaultValue?: LDAICompletionConfigDefault, + variables?: Record, + defaultAiProvider?: SupportedAIProvider, + ): Promise { + this._ldClient.track(TRACK_USAGE_CREATE_CHAT, context, key, 1); + const config = await this._completionConfig( + key, + context, + defaultValue ?? disabledAIConfig, + variables, + ); + + if (!config.enabled) { + this._logger?.info(`Chat configuration is disabled: ${key}`); + return undefined; + } + + const provider = await AIProviderFactory.create(config, this._logger, defaultAiProvider); + if (!provider) { + return undefined; + } + + const evaluator = await this._buildEvaluator( + config.judgeConfiguration?.judges ?? [], + context, + variables, + defaultAiProvider, + ); + + // Attach the evaluator to the config for use by the managed layer + const configWithEvaluator: LDAICompletionConfig = { ...config, evaluator }; + + const runner = runnerFromAIProvider(provider, configWithEvaluator); + return new ManagedModel(configWithEvaluator, runner, this._logger); + } + /** - * @deprecated Use `createChat` instead. This method will be removed in a future version. + * @deprecated Use `createModel` instead. This method will be removed in a future version. */ async initChat( key: string, @@ -419,8 +448,8 @@ export class LDAIClientImpl implements LDAIClient { defaultValue?: LDAICompletionConfigDefault, variables?: Record, defaultAiProvider?: SupportedAIProvider, - ): Promise { - return this.createChat(key, context, defaultValue, variables, defaultAiProvider); + ): Promise { + return this.createModel(key, context, defaultValue, variables, defaultAiProvider); } createTracker(token: string, context: LDContext): LDAIConfigTracker { diff --git a/packages/sdk/server-ai/src/api/LDAIClient.ts b/packages/sdk/server-ai/src/api/LDAIClient.ts index 5dfec98072..02dfb43e3d 100644 --- a/packages/sdk/server-ai/src/api/LDAIClient.ts +++ b/packages/sdk/server-ai/src/api/LDAIClient.ts @@ -1,6 +1,6 @@ import { LDContext } from '@launchdarkly/js-server-sdk-common'; -import { TrackedChat } from './chat'; +import { ManagedModel } from './chat'; import { LDAIAgentConfig, LDAIAgentConfigDefault, @@ -234,8 +234,7 @@ export interface LDAIClient { ): Promise>; /** - * Returns a TrackedChat instance for chat interactions. - * This method serves as the primary entry point for creating TrackedChat instances from configuration. + * Creates and returns a new ManagedModel instance for chat interactions. * * @param key The key identifying the AI chat configuration to use. * @param context The standard LDContext used when evaluating flags. @@ -245,7 +244,7 @@ export interface LDAIClient { * The variables will also be used for judge evaluation. For the judge only, the variables * `message_history` and `response_to_evaluate` are reserved and will be ignored. * @param defaultAiProvider Optional default AI provider to use. - * @returns A promise that resolves to the TrackedChat instance, or null if the configuration is disabled. + * @returns A promise that resolves to the ManagedModel instance, or undefined if the configuration is disabled. * * @example * ``` @@ -261,23 +260,34 @@ export interface LDAIClient { * }; * const variables = { customerName: 'John' }; * - * const chat = await client.createChat(key, context, defaultValue, variables); - * if (chat) { - * const response = await chat.invoke("I need help with my order"); - * console.log(response.message.content); + * const model = await client.createModel(key, context, defaultValue, variables); + * if (model) { + * const result = await model.run("I need help with my order"); + * console.log(result.content); * } * ``` */ + createModel( + key: string, + context: LDContext, + defaultValue?: LDAICompletionConfigDefault, + variables?: Record, + defaultAiProvider?: SupportedAIProvider, + ): Promise; + + /** + * @deprecated Use `createModel` instead. This method will be removed in a future version. + */ createChat( key: string, context: LDContext, defaultValue?: LDAICompletionConfigDefault, variables?: Record, defaultAiProvider?: SupportedAIProvider, - ): Promise; + ): Promise; /** - * @deprecated Use `createChat` instead. This method will be removed in a future version. + * @deprecated Use `createModel` instead. This method will be removed in a future version. */ initChat( key: string, @@ -285,7 +295,7 @@ export interface LDAIClient { defaultValue?: LDAICompletionConfigDefault, variables?: Record, defaultAiProvider?: SupportedAIProvider, - ): Promise; + ): Promise; /** * Creates and returns a new Judge instance for AI evaluation. diff --git a/packages/sdk/server-ai/src/api/chat/ManagedModel.ts b/packages/sdk/server-ai/src/api/chat/ManagedModel.ts new file mode 100644 index 0000000000..ef8a85a791 --- /dev/null +++ b/packages/sdk/server-ai/src/api/chat/ManagedModel.ts @@ -0,0 +1,69 @@ +import { LDLogger } from '@launchdarkly/js-server-sdk-common'; + +import { LDAICompletionConfig } from '../config/types'; +import { LDJudgeResult } from '../judge/types'; +import { LDAIMetricSummary, ManagedResult, RunnerResult } from '../model/types'; +import { Runner } from '../providers/Runner'; + +/** + * ManagedModel provides chat-completion invocation with automatic tracking and + * (in a future PR) automatic judge evaluation. + * + * The class is stateless: each `run()` call sends the prompt directly to the + * underlying `Runner` and returns a `ManagedResult`. Conversation history, + * if any, must be managed by the caller (or by the Runner implementation). + * + * Obtain an instance via `LDAIClient.createModel()`. + */ +export class ManagedModel { + constructor( + protected readonly aiConfig: LDAICompletionConfig, + protected readonly runner: Runner, + private readonly _logger?: LDLogger, + ) {} + + /** + * Invoke the model with a prompt string and return a ManagedResult. + * + * `run()` resolves before `ManagedResult.evaluations` resolves. Awaiting + * `evaluations` guarantees both judge evaluation and tracker.trackJudgeResult() + * are complete. + * + * @param prompt The user input to send to the model. + * @returns Promise resolving to ManagedResult (before evaluations settle). + */ + async run(prompt: string): Promise { + const tracker = this.aiConfig.createTracker!(); + + const result = await tracker.trackMetricsOf( + (r: RunnerResult) => r.metrics, + () => this.runner.run(prompt), + ); + + const metrics: LDAIMetricSummary = { + success: result.metrics.success, + usage: result.metrics.usage, + toolCalls: result.metrics.toolCalls, + durationMs: result.metrics.durationMs, + resumptionToken: tracker.resumptionToken, + }; + + // Evaluations are wired in a follow-up PR. For now, resolve empty. + const evaluations: Promise = Promise.resolve([]); + + return { + content: result.content, + metrics, + raw: result.raw, + parsed: result.parsed, + evaluations, + }; + } + + /** + * Get the underlying AI configuration used to initialize this ManagedModel. + */ + getConfig(): LDAICompletionConfig { + return this.aiConfig; + } +} diff --git a/packages/sdk/server-ai/src/api/chat/TrackedChat.ts b/packages/sdk/server-ai/src/api/chat/TrackedChat.ts deleted file mode 100644 index 2d5b21a85f..0000000000 --- a/packages/sdk/server-ai/src/api/chat/TrackedChat.ts +++ /dev/null @@ -1,167 +0,0 @@ -import { LDLogger } from '@launchdarkly/js-server-sdk-common'; - -import { LDAICompletionConfig, LDMessage } from '../config/types'; -import { Judge } from '../judge/Judge'; -import { LDJudgeResult } from '../judge/types'; -import { AIProvider } from '../providers/AIProvider'; -import { ChatResponse } from './types'; - -/** - * Concrete implementation of TrackedChat that provides chat functionality - * by delegating to an AIProvider implementation. - * This class handles conversation management and tracking, while delegating - * the actual model invocation to the provider. - */ -export class TrackedChat { - protected messages: LDMessage[]; - - constructor( - protected readonly aiConfig: LDAICompletionConfig, - protected readonly provider: AIProvider, - protected readonly judges: Record = {}, - private readonly _logger?: LDLogger, - ) { - this.messages = []; - } - - /** - * Invoke the chat model with a prompt string. - * This method handles conversation management and tracking, delegating to the provider's invokeModel method. - */ - async invoke(prompt: string): Promise { - const tracker = this.aiConfig.createTracker!(); - - // Convert prompt string to LDMessage with role 'user' and add to conversation history - const userMessage: LDMessage = { - role: 'user', - content: prompt, - }; - this.messages.push(userMessage); - - // Prepend config messages to conversation history for model invocation - const configMessages = this.aiConfig.messages || []; - const allMessages = [...configMessages, ...this.messages]; - - // Delegate to provider-specific implementation with tracking - const response = await tracker.trackMetricsOf( - (result: ChatResponse) => result.metrics, - () => this.provider.invokeModel(allMessages), - ); - - if ( - this.aiConfig.judgeConfiguration?.judges && - this.aiConfig.judgeConfiguration.judges.length > 0 - ) { - response.evaluations = this._evaluateWithJudges(this.messages, response).then( - (evaluations) => { - evaluations.forEach((judgeResult) => { - tracker.trackJudgeResult(judgeResult); - }); - return evaluations; - }, - ); - } - - this.messages.push(response.message); - return response; - } - - /** - * Evaluates the response with all configured judges. - * Returns a promise that resolves to an array of evaluation results. - * - * @param messages Array of messages representing the conversation history - * @param response The AI response to be evaluated - * @returns Promise resolving to array of judge evaluation results - */ - private async _evaluateWithJudges( - messages: LDMessage[], - response: ChatResponse, - ): Promise { - const judgeConfigs = this.aiConfig.judgeConfiguration!.judges; - - // Start all judge evaluations in parallel - const evaluationPromises = judgeConfigs.map(async (judgeConfig) => { - const judge = this.judges[judgeConfig.key]; - if (!judge) { - this._logger?.warn( - `Judge configuration is not enabled for ${judgeConfig.key} in ${this.aiConfig.key}`, - ); - const result: LDJudgeResult = { - success: false, - sampled: true, - errorMessage: `Judge configuration is not enabled for ${judgeConfig.key}`, - }; - return result; - } - - return judge.evaluateMessages(messages, response, judgeConfig.samplingRate); - }); - - // ensure all evaluations complete even if some fail - const results = await Promise.allSettled(evaluationPromises); - - return results.map((settled) => { - if (settled.status === 'fulfilled') { - return settled.value; - } - const result: LDJudgeResult = { - success: false, - sampled: true, - errorMessage: 'Judge evaluation failed', - }; - return result; - }); - } - - /** - * Get the underlying AI configuration used to initialize this TrackedChat. - */ - getConfig(): LDAICompletionConfig { - return this.aiConfig; - } - - /** - * Get the underlying AI provider instance. - * This provides direct access to the provider for advanced use cases. - */ - getProvider(): AIProvider { - return this.provider; - } - - /** - * Get the judges associated with this TrackedChat. - * Returns a record of judge instances keyed by their configuration keys. - */ - getJudges(): Record { - return this.judges; - } - - /** - * Append messages to the conversation history. - * Adds messages to the conversation history without invoking the model, - * which is useful for managing multi-turn conversations or injecting context. - * - * @param messages Array of messages to append to the conversation history - */ - appendMessages(messages: LDMessage[]): void { - this.messages.push(...messages); - } - - /** - * Get all messages in the conversation history. - * - * @param includeConfigMessages Whether to include the config messages from the AIConfig. - * Defaults to false. - * @returns Array of messages. When includeConfigMessages is true, returns both config - * messages and conversation history with config messages prepended. When false, - * returns only the conversation history messages. - */ - getMessages(includeConfigMessages: boolean = false): LDMessage[] { - if (includeConfigMessages) { - const configMessages = this.aiConfig.messages || []; - return [...configMessages, ...this.messages]; - } - return [...this.messages]; - } -} diff --git a/packages/sdk/server-ai/src/api/chat/index.ts b/packages/sdk/server-ai/src/api/chat/index.ts index f7876298ea..8266851b09 100644 --- a/packages/sdk/server-ai/src/api/chat/index.ts +++ b/packages/sdk/server-ai/src/api/chat/index.ts @@ -1,2 +1,2 @@ export * from './types'; -export * from './TrackedChat'; +export * from './ManagedModel'; diff --git a/packages/sdk/server-ai/src/api/graph/types.ts b/packages/sdk/server-ai/src/api/graph/types.ts index 1b578fecba..6cca861c25 100644 --- a/packages/sdk/server-ai/src/api/graph/types.ts +++ b/packages/sdk/server-ai/src/api/graph/types.ts @@ -1,4 +1,4 @@ -import { LDTokenUsage } from '../metrics'; +import { LDAIMetrics, LDTokenUsage } from '../metrics'; /** * Represents a directed edge in an agent graph, connecting a source node to a target node. @@ -62,6 +62,58 @@ export interface LDGraphMetricSummary { path?: string[]; } +/** + * Graph-level metrics for a completed graph run, as returned by a graph runner. + * Does NOT include handoffs or evaluations — those are managed-layer concerns. + */ +export interface GraphMetrics { + /** + * Whether the graph invocation succeeded. + */ + success: boolean; + + /** + * Execution path through the graph as an ordered array of config keys. + */ + path: string[]; + + /** + * Total graph execution duration in milliseconds, if tracked. + */ + durationMs?: number; + + /** + * Aggregate token usage across the entire graph invocation, if available. + */ + usage?: LDTokenUsage; + + /** + * Per-node metrics keyed by agent config key. + */ + nodeMetrics: Record; +} + +/** + * The result returned by a graph runner invocation (provider-level). + * Does NOT include evaluations or handoffs. + */ +export interface AgentGraphRunnerResult { + /** + * The text content of the graph's final response. + */ + content: string; + + /** + * Graph-level metrics for this invocation. + */ + metrics: GraphMetrics; + + /** + * The raw response object from the provider, if available. + */ + raw?: unknown; +} + /** * Tracking metadata returned by {@link LDGraphTracker.getTrackData}. */ diff --git a/packages/sdk/server-ai/src/api/index.ts b/packages/sdk/server-ai/src/api/index.ts index 7470ef740c..53801ec6d1 100644 --- a/packages/sdk/server-ai/src/api/index.ts +++ b/packages/sdk/server-ai/src/api/index.ts @@ -3,5 +3,6 @@ export * from './chat'; export * from './graph'; export * from './judge'; export * from './metrics'; +export * from './model'; export * from './LDAIClient'; export * from './providers'; diff --git a/packages/sdk/server-ai/src/api/metrics/LDAIMetrics.ts b/packages/sdk/server-ai/src/api/metrics/LDAIMetrics.ts index 3b0fb99ec7..2d9f3a47d7 100644 --- a/packages/sdk/server-ai/src/api/metrics/LDAIMetrics.ts +++ b/packages/sdk/server-ai/src/api/metrics/LDAIMetrics.ts @@ -15,4 +15,16 @@ export interface LDAIMetrics { * This will be undefined if no token usage data is available. */ usage?: LDTokenUsage; + + /** + * List of tool call identifiers made during the operation. + * This will be undefined if no tool calls were made. + */ + toolCalls?: string[]; + + /** + * Duration of the operation in milliseconds. + * This will be undefined if duration was not tracked. + */ + durationMs?: number; } diff --git a/packages/sdk/server-ai/src/api/model/index.ts b/packages/sdk/server-ai/src/api/model/index.ts new file mode 100644 index 0000000000..3869bd6bcd --- /dev/null +++ b/packages/sdk/server-ai/src/api/model/index.ts @@ -0,0 +1 @@ +export type { LDAIMetricSummary, ManagedResult, RunnerResult } from './types'; diff --git a/packages/sdk/server-ai/src/api/model/types.ts b/packages/sdk/server-ai/src/api/model/types.ts new file mode 100644 index 0000000000..e92d189288 --- /dev/null +++ b/packages/sdk/server-ai/src/api/model/types.ts @@ -0,0 +1,95 @@ +import { LDJudgeResult } from '../judge/types'; +import { LDAIMetrics } from '../metrics/LDAIMetrics'; +import { LDTokenUsage } from '../metrics/LDTokenUsage'; + +/** + * Summary metrics returned in a ManagedResult or ManagedGraphResult. + * Provides a flat view of the key metrics for the completed operation. + */ +export interface LDAIMetricSummary { + /** + * Whether the AI operation was successful. + */ + success: boolean; + + /** + * Token usage information, if available. + */ + usage?: LDTokenUsage; + + /** + * List of tool call identifiers made during the operation, if any. + */ + toolCalls?: string[]; + + /** + * Duration of the operation in milliseconds, if tracked. + */ + durationMs?: number; + + /** + * Resumption token for deferred feedback association. + */ + resumptionToken?: string; +} + +/** + * The result returned by a Runner (provider-level) invocation. + * Providers implement Runner and return RunnerResult from run(). + * This type does NOT include evaluations — those are wired in the managed layer. + */ +export interface RunnerResult { + /** + * The text content of the model's response. + */ + content: string; + + /** + * Metrics information for the operation. + */ + metrics: LDAIMetrics; + + /** + * The raw response object from the provider, if available. + */ + raw?: unknown; + + /** + * Parsed structured output, if the provider returned structured data. + */ + parsed?: Record; +} + +/** + * The result returned by a managed model invocation (ManagedModel.run()). + * Includes a promise for asynchronous judge evaluations. + */ +export interface ManagedResult { + /** + * The text content of the model's response. + */ + content: string; + + /** + * Summarized metrics for this invocation. + */ + metrics: LDAIMetricSummary; + + /** + * The raw response object from the provider, if available. + */ + raw?: unknown; + + /** + * Parsed structured output, if available. + */ + parsed?: Record; + + /** + * Promise that resolves to the judge evaluation results. + * This promise encapsulates both evaluation and tracking + * (tracker.trackJudgeResult is called when it resolves). + * Awaiting this promise guarantees both evaluation and tracking are complete. + */ + evaluations: Promise; +} diff --git a/packages/sdk/server-ai/src/api/providers/AIProvider.ts b/packages/sdk/server-ai/src/api/providers/AIProvider.ts index e83ea2a834..8e0f7a098a 100644 --- a/packages/sdk/server-ai/src/api/providers/AIProvider.ts +++ b/packages/sdk/server-ai/src/api/providers/AIProvider.ts @@ -11,6 +11,10 @@ import { StructuredResponse } from '../judge/types'; * * Following the AICHAT spec recommendation to use base classes with non-abstract methods * for better extensibility and backwards compatibility. + * + * @deprecated Use the `Runner` interface instead. Provider implementations should + * implement `Runner` (and optionally `AgentGraphRunner`) rather than extending this + * abstract class. This class will be removed in a future major version. */ export abstract class AIProvider { protected readonly logger?: LDLogger; diff --git a/packages/sdk/server-ai/src/api/providers/Runner.ts b/packages/sdk/server-ai/src/api/providers/Runner.ts new file mode 100644 index 0000000000..ce4bea0ec9 --- /dev/null +++ b/packages/sdk/server-ai/src/api/providers/Runner.ts @@ -0,0 +1,40 @@ +import { AgentGraphRunnerResult } from '../graph/types'; +import { RunnerResult } from '../model/types'; + +/** + * Runner protocol for AI model providers. + * + * Providers implementing the Runner interface can be used with ManagedModel + * and ManagedAgent without extending the deprecated AIProvider abstract class. + * + * A single Runner interface covers both chat (completion) and agent use cases. + * For structured output (e.g., judge evaluation), pass an `outputType` schema + * and access the parsed result via `RunnerResult.parsed`. + */ +export interface Runner { + /** + * Invoke the model with a prompt string. + * + * @param input The prompt to send to the model. + * @param outputType Optional JSON schema for structured output. When provided, + * the model should return structured data accessible via `RunnerResult.parsed`. + * @returns Promise resolving to a RunnerResult. + */ + run(input: string, outputType?: Record): Promise; +} + +/** + * Runner protocol for agent graph providers. + * + * Providers implementing AgentGraphRunner can execute an entire agent graph + * and return a structured AgentGraphRunnerResult. + */ +export interface AgentGraphRunner { + /** + * Execute the agent graph with the given input. + * + * @param input The user input to process through the graph. + * @returns Promise resolving to an AgentGraphRunnerResult. + */ + run(input: string): Promise; +} diff --git a/packages/sdk/server-ai/src/api/providers/index.ts b/packages/sdk/server-ai/src/api/providers/index.ts index 5439dfa830..f80a1db862 100644 --- a/packages/sdk/server-ai/src/api/providers/index.ts +++ b/packages/sdk/server-ai/src/api/providers/index.ts @@ -1,2 +1,3 @@ export * from './AIProvider'; export * from './AIProviderFactory'; +export * from './Runner';