diff --git a/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts b/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts index 9695c1f815..77af66a0b5 100644 --- a/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts +++ b/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts @@ -139,7 +139,14 @@ describe('config evaluation', () => { const evaluateSpy = jest.spyOn(client as any, '_evaluate'); const result = await client.agentConfig(key, testContext, defaultValue, variables); - expect(evaluateSpy).toHaveBeenCalledWith(key, testContext, defaultValue, 'agent', variables); + expect(evaluateSpy).toHaveBeenCalledWith( + key, + testContext, + defaultValue, + 'agent', + variables, + undefined, + ); expect(result.instructions).toBe( 'You are a helpful assistant. Your name is John and your score is 42', ); @@ -464,7 +471,14 @@ describe('agentConfig method', () => { key, 1, ); - expect(evaluateSpy).toHaveBeenCalledWith(key, testContext, defaultValue, 'agent', variables); + expect(evaluateSpy).toHaveBeenCalledWith( + key, + testContext, + defaultValue, + 'agent', + variables, + undefined, + ); expect(result).toBe(mockConfig); evaluateSpy.mockRestore(); }); diff --git a/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts b/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts index e644eff377..a4b40b62cb 100644 --- a/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts +++ b/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts @@ -933,7 +933,7 @@ describe('trackToolCall', () => { ); }); - it('includes graphKey when provided', () => { + it('includes graphKey when set on constructor', () => { const tracker = new LDAIConfigTrackerImpl( mockLdClient, testRunId, @@ -943,9 +943,10 @@ describe('trackToolCall', () => { modelName, providerName, testContext, + 'my-graph', ); - tracker.trackToolCall('my-tool', 'my-graph'); + tracker.trackToolCall('my-tool'); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tool_call', @@ -993,8 +994,8 @@ describe('trackToolCalls', () => { }); }); -describe('graphKey parameter support', () => { - it('includes graphKey in trackDuration event', () => { +describe('graphKey constructor support', () => { + it('includes graphKey in trackDuration event when set on constructor', () => { const tracker = new LDAIConfigTrackerImpl( mockLdClient, testRunId, @@ -1004,9 +1005,10 @@ describe('graphKey parameter support', () => { modelName, providerName, testContext, + 'my-graph', ); - tracker.trackDuration(1000, 'my-graph'); + tracker.trackDuration(1000); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', @@ -1016,7 +1018,7 @@ describe('graphKey parameter support', () => { ); }); - it('includes graphKey in trackSuccess event', () => { + it('includes graphKey in trackSuccess event when set on constructor', () => { const tracker = new LDAIConfigTrackerImpl( mockLdClient, testRunId, @@ -1026,9 +1028,10 @@ describe('graphKey parameter support', () => { modelName, providerName, testContext, + 'my-graph', ); - tracker.trackSuccess('my-graph'); + tracker.trackSuccess(); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:success', @@ -1038,7 +1041,7 @@ describe('graphKey parameter support', () => { ); }); - it('does not include graphKey when not provided', () => { + it('does not include graphKey when not set on constructor', () => { const tracker = new LDAIConfigTrackerImpl( mockLdClient, testRunId, @@ -1059,6 +1062,41 @@ describe('graphKey parameter support', () => { 1, ); }); + + it('includes graphKey in getTrackData when set on constructor', () => { + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + testRunId, + configKey, + variationKey, + version, + modelName, + providerName, + testContext, + 'my-graph', + ); + + expect(tracker.getTrackData()).toEqual({ + ...getExpectedTrackData(), + graphKey: 'my-graph', + }); + }); + + it('does not include graphKey in getTrackData when not set', () => { + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + testRunId, + configKey, + variationKey, + version, + modelName, + providerName, + testContext, + ); + + expect(tracker.getTrackData()).toEqual(getExpectedTrackData()); + expect('graphKey' in tracker.getTrackData()).toBe(false); + }); }); describe('at-most-once semantics', () => { @@ -1311,4 +1349,96 @@ describe('fromResumptionToken', () => { 1, ); }); + + it('includes graphKey in resumption token when set on constructor', () => { + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + testRunId, + configKey, + variationKey, + version, + modelName, + providerName, + testContext, + 'my-graph', + ); + + const token = tracker.resumptionToken; + const decoded = JSON.parse(Buffer.from(token, 'base64url').toString('utf8')); + + expect(decoded).toEqual({ + runId: testRunId, + configKey, + variationKey, + version, + graphKey: 'my-graph', + }); + }); + + it('does not include graphKey in resumption token when not set', () => { + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + testRunId, + configKey, + variationKey, + version, + modelName, + providerName, + testContext, + ); + + const token = tracker.resumptionToken; + const decoded = JSON.parse(Buffer.from(token, 'base64url').toString('utf8')); + + expect(decoded).toEqual({ + runId: testRunId, + configKey, + variationKey, + version, + }); + expect('graphKey' in decoded).toBe(false); + }); + + it('reconstructs tracker with graphKey from resumption token', () => { + const original = new LDAIConfigTrackerImpl( + mockLdClient, + testRunId, + configKey, + variationKey, + version, + modelName, + providerName, + testContext, + 'my-graph', + ); + + const reconstructed = LDAIConfigTrackerImpl.fromResumptionToken( + original.resumptionToken, + mockLdClient, + testContext, + ); + + expect(reconstructed.getTrackData().graphKey).toBe('my-graph'); + }); + + it('reconstructed tracker without graphKey does not include graphKey in track data', () => { + const original = new LDAIConfigTrackerImpl( + mockLdClient, + testRunId, + configKey, + variationKey, + version, + modelName, + providerName, + testContext, + ); + + const reconstructed = LDAIConfigTrackerImpl.fromResumptionToken( + original.resumptionToken, + mockLdClient, + testContext, + ); + + expect('graphKey' in reconstructed.getTrackData()).toBe(false); + }); }); diff --git a/packages/sdk/server-ai/src/LDAIClientImpl.ts b/packages/sdk/server-ai/src/LDAIClientImpl.ts index 209c0ce860..65eb87a1a9 100644 --- a/packages/sdk/server-ai/src/LDAIClientImpl.ts +++ b/packages/sdk/server-ai/src/LDAIClientImpl.ts @@ -74,6 +74,7 @@ export class LDAIClientImpl implements LDAIClient { defaultValue: LDAIConfigDefaultKind, mode: LDAIConfigMode, variables?: Record, + graphKey?: string, ): Promise { const ldFlagValue = LDAIConfigUtils.toFlagValue(defaultValue, mode); @@ -101,6 +102,7 @@ export class LDAIClientImpl implements LDAIClient { value.model?.name ?? '', value.provider?.name ?? '', context, + graphKey, ); const config = LDAIConfigUtils.fromFlagValue(key, value, trackerFactory); @@ -217,6 +219,17 @@ export class LDAIClientImpl implements LDAIClient { return this._judgeConfig(key, context, defaultValue ?? disabledAIConfig, variables); } + private async _agentConfig( + key: string, + context: LDContext, + defaultValue: LDAIAgentConfigDefault, + variables?: Record, + graphKey?: string, + ): Promise { + const config = await this._evaluate(key, context, defaultValue, 'agent', variables, graphKey); + return config as LDAIAgentConfig; + } + async agentConfig( key: string, context: LDContext, @@ -224,14 +237,7 @@ export class LDAIClientImpl implements LDAIClient { variables?: Record, ): Promise { this._ldClient.track(TRACK_USAGE_AGENT_CONFIG, context, key, 1); - const config = await this._evaluate( - key, - context, - defaultValue ?? disabledAIConfig, - 'agent', - variables, - ); - return config as LDAIAgentConfig; + return this._agentConfig(key, context, defaultValue ?? disabledAIConfig, variables); } /** @@ -261,14 +267,13 @@ export class LDAIClientImpl implements LDAIClient { await Promise.all( agentConfigs.map(async (config) => { - const agent = await this._evaluate( + const agent = await this._agentConfig( config.key, context, config.defaultValue ?? disabledAIConfig, - 'agent', config.variables, ); - agents[config.key as T[number]['key']] = agent as LDAIAgentConfig; + agents[config.key as T[number]['key']] = agent; }), ); diff --git a/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts b/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts index 151a3c1d97..d87729c14f 100644 --- a/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts +++ b/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts @@ -25,9 +25,10 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { private _modelName: string, private _providerName: string, private _context: LDContext, + private _graphKey?: string, ) {} - getTrackData(graphKey?: string): { + getTrackData(): { runId: string; configKey: string; variationKey: string; @@ -43,7 +44,7 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { version: this._version, modelName: this._modelName, providerName: this._providerName, - ...(graphKey !== undefined ? { graphKey } : {}), + ...(this._graphKey !== undefined ? { graphKey: this._graphKey } : {}), }; } @@ -53,6 +54,7 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { configKey: this._configKey, variationKey: this._variationKey, version: this._version, + ...(this._graphKey !== undefined ? { graphKey: this._graphKey } : {}), }); return Buffer.from(json).toString('base64url'); } @@ -73,10 +75,11 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { '', '', context, + payload.graphKey, ); } - trackDuration(duration: number, graphKey?: string): void { + trackDuration(duration: number): void { if (this._trackedMetrics.durationMs !== undefined) { this._ldClient.logger?.warn( 'Duration has already been tracked for this execution. Use createTracker() for a new execution.', @@ -84,15 +87,10 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { return; } this._trackedMetrics.durationMs = duration; - this._ldClient.track( - '$ld:ai:duration:total', - this._context, - this.getTrackData(graphKey), - duration, - ); + this._ldClient.track('$ld:ai:duration:total', this._context, this.getTrackData(), duration); } - async trackDurationOf(func: () => Promise, graphKey?: string): Promise { + async trackDurationOf(func: () => Promise): Promise { const startTime = Date.now(); try { // Be sure to await here so that we can track the duration of the function and also handle errors. @@ -101,11 +99,11 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { } finally { const endTime = Date.now(); const duration = endTime - startTime; // duration in milliseconds - this.trackDuration(duration, graphKey); + this.trackDuration(duration); } } - trackTimeToFirstToken(timeToFirstTokenMs: number, graphKey?: string) { + trackTimeToFirstToken(timeToFirstTokenMs: number) { if (this._trackedMetrics.timeToFirstTokenMs !== undefined) { this._ldClient.logger?.warn( 'Time to first token has already been tracked for this execution. Use createTracker() for a new execution.', @@ -116,44 +114,39 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { this._ldClient.track( '$ld:ai:tokens:ttf', this._context, - this.getTrackData(graphKey), + this.getTrackData(), timeToFirstTokenMs, ); } - trackEvalScores(scores: Record, graphKey?: string) { + trackEvalScores(scores: Record) { Object.entries(scores).forEach(([metricKey, evalScore]) => { - this._ldClient.track(metricKey, this._context, this.getTrackData(graphKey), evalScore.score); + this._ldClient.track(metricKey, this._context, this.getTrackData(), evalScore.score); }); } - trackJudgeResponse(response: JudgeResponse, graphKey?: string) { + trackJudgeResponse(response: JudgeResponse) { Object.entries(response.evals).forEach(([metricKey, evalScore]) => { this._ldClient.track( metricKey, this._context, - { ...this.getTrackData(graphKey), judgeConfigKey: response.judgeConfigKey }, + { ...this.getTrackData(), judgeConfigKey: response.judgeConfigKey }, evalScore.score, ); }); } - trackToolCall(toolKey: string, graphKey?: string): void { - this._ldClient.track( - '$ld:ai:tool_call', - this._context, - { ...this.getTrackData(graphKey), toolKey }, - 1, - ); + trackToolCall(toolKey: string): void { + this._ldClient.track('$ld:ai:tool_call', this._context, { ...this.getTrackData(), toolKey }, 1); } - trackToolCalls(toolKeys: string[], graphKey?: string): void { + trackToolCalls(toolKeys: string[]): void { toolKeys.forEach((toolKey) => { - this.trackToolCall(toolKey, graphKey); + this.trackToolCall(toolKey); }); } - trackFeedback(feedback: { kind: LDFeedbackKind }, graphKey?: string): void { + trackFeedback(feedback: { kind: LDFeedbackKind }): void { if (this._trackedMetrics.feedback !== undefined) { this._ldClient.logger?.warn( 'Feedback has already been tracked for this execution. Use createTracker() for a new execution.', @@ -162,23 +155,13 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { } this._trackedMetrics.feedback = feedback; if (feedback.kind === LDFeedbackKind.Positive) { - this._ldClient.track( - '$ld:ai:feedback:user:positive', - this._context, - this.getTrackData(graphKey), - 1, - ); + this._ldClient.track('$ld:ai:feedback:user:positive', this._context, this.getTrackData(), 1); } else if (feedback.kind === LDFeedbackKind.Negative) { - this._ldClient.track( - '$ld:ai:feedback:user:negative', - this._context, - this.getTrackData(graphKey), - 1, - ); + this._ldClient.track('$ld:ai:feedback:user:negative', this._context, this.getTrackData(), 1); } } - trackSuccess(graphKey?: string): void { + trackSuccess(): void { if (this._trackedMetrics.success !== undefined) { this._ldClient.logger?.warn( 'Generation result has already been tracked for this execution. Use createTracker() for a new execution.', @@ -186,15 +169,10 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { return; } this._trackedMetrics.success = true; - this._ldClient.track( - '$ld:ai:generation:success', - this._context, - this.getTrackData(graphKey), - 1, - ); + this._ldClient.track('$ld:ai:generation:success', this._context, this.getTrackData(), 1); } - trackError(graphKey?: string): void { + trackError(): void { if (this._trackedMetrics.success !== undefined) { this._ldClient.logger?.warn( 'Generation result has already been tracked for this execution. Use createTracker() for a new execution.', @@ -202,20 +180,19 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { return; } this._trackedMetrics.success = false; - this._ldClient.track('$ld:ai:generation:error', this._context, this.getTrackData(graphKey), 1); + this._ldClient.track('$ld:ai:generation:error', this._context, this.getTrackData(), 1); } async trackMetricsOf( metricsExtractor: (result: TRes) => LDAIMetrics, func: () => Promise, - graphKey?: string, ): Promise { let result: TRes; try { - result = await this.trackDurationOf(func, graphKey); + result = await this.trackDurationOf(func); } catch (err) { - this.trackError(graphKey); + this.trackError(); throw err; } @@ -224,14 +201,14 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { // Track success/error based on metrics if (metrics.success) { - this.trackSuccess(graphKey); + this.trackSuccess(); } else { - this.trackError(graphKey); + this.trackError(); } // Track token usage if available if (metrics.usage) { - this.trackTokens(metrics.usage, graphKey); + this.trackTokens(metrics.usage); } return result; @@ -240,7 +217,6 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { trackStreamMetricsOf( streamCreator: () => TStream, metricsExtractor: (stream: TStream) => Promise, - graphKey?: string, ): TStream { const startTime = Date.now(); @@ -249,14 +225,14 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { const stream = streamCreator(); // Start background metrics tracking (fire and forget) - this._trackStreamMetricsInBackground(stream, metricsExtractor, startTime, graphKey); + this._trackStreamMetricsInBackground(stream, metricsExtractor, startTime); // Return stream immediately for consumption return stream; } catch (error) { // Track error if stream creation fails - this.trackDuration(Date.now() - startTime, graphKey); - this.trackError(graphKey); + this.trackDuration(Date.now() - startTime); + this.trackError(); throw error; } } @@ -265,7 +241,6 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { stream: TStream, metricsExtractor: (stream: TStream) => Promise, startTime: number, - graphKey?: string, ): Promise { try { // Wait for metrics to be available @@ -273,21 +248,21 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { // Track success/error based on metrics if (metrics.success) { - this.trackSuccess(graphKey); + this.trackSuccess(); } else { - this.trackError(graphKey); + this.trackError(); } // Track token usage if available if (metrics.usage) { - this.trackTokens(metrics.usage, graphKey); + this.trackTokens(metrics.usage); } } catch (error) { // If metrics extraction fails, track error - this.trackError(graphKey); + this.trackError(); } finally { // Track duration regardless of success/error - this.trackDuration(Date.now() - startTime, graphKey); + this.trackDuration(Date.now() - startTime); } } @@ -362,7 +337,7 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { } } - trackTokens(tokens: LDTokenUsage, graphKey?: string): void { + trackTokens(tokens: LDTokenUsage): void { if (this._trackedMetrics.tokens !== undefined) { this._ldClient.logger?.warn( 'Token usage has already been tracked for this execution. Use createTracker() for a new execution.', @@ -370,7 +345,7 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { return; } this._trackedMetrics.tokens = tokens; - const trackData = this.getTrackData(graphKey); + const trackData = this.getTrackData(); if (tokens.total > 0) { this._ldClient.track('$ld:ai:tokens:total', this._context, trackData, tokens.total); } diff --git a/packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts b/packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts index 18b243d94b..883177becb 100644 --- a/packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts +++ b/packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts @@ -37,10 +37,8 @@ export interface LDAIMetricSummary { export interface LDAIConfigTracker { /** * Get the data for tracking. - * - * @param graphKey When provided, associates this metric with the specified agent graph key. */ - getTrackData(graphKey?: string): { + getTrackData(): { runId: string; configKey: string; variationKey: string; @@ -68,9 +66,8 @@ export interface LDAIConfigTracker { * Ideally this would not include overhead time such as network communication. * * @param durationMs The duration in milliseconds. - * @param graphKey When provided, associates this metric with the specified agent graph key. */ - trackDuration(durationMs: number, graphKey?: string): void; + trackDuration(durationMs: number): void; /** * Track information about token usage. @@ -79,29 +76,24 @@ export interface LDAIConfigTracker { * with a warning. * * @param tokens Token usage information. - * @param graphKey When provided, associates this metric with the specified agent graph key. */ - trackTokens(tokens: LDTokenUsage, graphKey?: string): void; + trackTokens(tokens: LDTokenUsage): void; /** * Generation was successful. * * At-most-once per execution: subsequent calls (including trackError) on the * same tracker are dropped with a warning. - * - * @param graphKey When provided, associates this metric with the specified agent graph key. */ - trackSuccess(graphKey?: string): void; + trackSuccess(): void; /** * An error was encountered during generation. * * At-most-once per execution: subsequent calls (including trackSuccess) on the * same tracker are dropped with a warning. - * - * @param graphKey When provided, associates this metric with the specified agent graph key. */ - trackError(graphKey?: string): void; + trackError(): void; /** * Track sentiment about the generation. @@ -110,9 +102,8 @@ export interface LDAIConfigTracker { * with a warning. * * @param feedback Feedback about the generation. - * @param graphKey When provided, associates this metric with the specified agent graph key. */ - trackFeedback(feedback: { kind: LDFeedbackKind }, graphKey?: string): void; + trackFeedback(feedback: { kind: LDFeedbackKind }): void; /** * Track the time to first token for this generation. @@ -121,41 +112,36 @@ export interface LDAIConfigTracker { * with a warning. * * @param timeToFirstTokenMs The duration in milliseconds. - * @param graphKey When provided, associates this metric with the specified agent graph key. */ - trackTimeToFirstToken(timeToFirstTokenMs: number, graphKey?: string): void; + trackTimeToFirstToken(timeToFirstTokenMs: number): void; /** * Track evaluation scores for multiple metrics. * * @param scores Record mapping metric keys to their evaluation scores - * @param graphKey When provided, associates this metric with the specified agent graph key. */ - trackEvalScores(scores: Record, graphKey?: string): void; + trackEvalScores(scores: Record): void; /** * Track a judge response containing evaluation scores and judge configuration key. * * @param response Judge response containing evaluation scores and judge configuration key - * @param graphKey When provided, associates this metric with the specified agent graph key. */ - trackJudgeResponse(response: JudgeResponse, graphKey?: string): void; + trackJudgeResponse(response: JudgeResponse): void; /** * Track a single tool invocation. * * @param toolKey The identifier of the tool that was invoked. - * @param graphKey When provided, associates this metric with the specified agent graph key. */ - trackToolCall(toolKey: string, graphKey?: string): void; + trackToolCall(toolKey: string): void; /** * Track multiple tool invocations. * * @param toolKeys The identifiers of the tools that were invoked. - * @param graphKey When provided, associates this metric with the specified agent graph key. */ - trackToolCalls(toolKeys: string[], graphKey?: string): void; + trackToolCalls(toolKeys: string[]): void; /** * Track the duration of execution of the provided function. @@ -166,10 +152,9 @@ export interface LDAIConfigTracker { * This function does not automatically record an error when the function throws. * * @param func The function to track the duration of. - * @param graphKey When provided, associates this metric with the specified agent graph key. * @returns The result of the function. */ - trackDurationOf(func: () => Promise, graphKey?: string): Promise; + trackDurationOf(func: () => Promise): Promise; /** * Track metrics for a generic AI operation. @@ -183,13 +168,11 @@ export interface LDAIConfigTracker { * * @param metricsExtractor Function that extracts LDAIMetrics from the operation result * @param func Function which executes the operation - * @param graphKey When provided, associates this metric with the specified agent graph key. * @returns The result of the operation */ trackMetricsOf( metricsExtractor: (result: TRes) => LDAIMetrics, func: () => Promise, - graphKey?: string, ): Promise; /** @@ -211,13 +194,11 @@ export interface LDAIConfigTracker { * * @param streamCreator Function that creates and returns the stream (synchronous) * @param metricsExtractor Function that asynchronously extracts metrics from the stream - * @param graphKey When provided, associates this metric with the specified agent graph key. * @returns The stream result (returned immediately, not a Promise) */ trackStreamMetricsOf( streamCreator: () => TStream, metricsExtractor: (stream: TStream) => Promise, - graphKey?: string, ): TStream; /**