Skip to content

docs: Clarify runId purpose and per-method tracker semantics #1378

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jsonbailey merged 3 commits into from
May 14, 2026
Merged

docs: Clarify runId purpose and per-method tracker semantics #1378

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2f8a149
refactor: Clarify already-tracked warnings as skips with remedy
jsonbailey May 13, 2026
0afe9da
docs: Clarify runId purpose and per-method tracker semantics
jsonbailey May 13, 2026
1407150
docs: Avoid "run...runId" duplicate phrasing in resumption docs
jsonbailey May 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -973,7 +973,7 @@ describe('at-most-once semantics', () => {

expect(mockTrack).toHaveBeenCalledTimes(1);
expect(mockWarn).toHaveBeenCalledTimes(1);
expect(mockWarn).toHaveBeenCalledWith(expect.stringContaining('Duration'));
expect(mockWarn).toHaveBeenCalledWith(expect.stringContaining('trackDuration'));
Comment thread
jsonbailey marked this conversation as resolved.
});

it('drops duplicate trackSuccess call with warning', () => {
Expand Down
4 changes: 2 additions & 2 deletions packages/sdk/server-ai/__tests__/LDGraphTrackerImpl.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ it('drops second trackInvocationSuccess call and warns', () => {
tracker.trackInvocationSuccess();
expect(mockTrack).toHaveBeenCalledTimes(1);
expect(mockWarn).toHaveBeenCalledWith(
expect.stringContaining('invocation success/failure already recorded for this run'),
expect.stringContaining('invocation result already recorded on this graph tracker'),
);
});

Expand All @@ -161,7 +161,7 @@ it('drops trackInvocationFailure after trackInvocationSuccess and warns', () =>
tracker.trackInvocationFailure();
expect(mockTrack).toHaveBeenCalledTimes(1);
expect(mockWarn).toHaveBeenCalledWith(
expect.stringContaining('invocation success/failure already recorded for this run'),
expect.stringContaining('invocation result already recorded on this graph tracker'),
);
});

Expand Down
12 changes: 6 additions & 6 deletions packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker {
trackDuration(duration: number): void {
if (this._trackedMetrics.durationMs !== undefined) {
this._ldClient.logger?.warn(
'Duration has already been tracked for this execution. Use createTracker() for a new execution.',
'Skipping trackDuration: duration already recorded on this tracker. Call createTracker on the AI Config for a new run.',
);
return;
}
Expand All @@ -106,7 +106,7 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker {
trackTimeToFirstToken(timeToFirstTokenMs: number) {
if (this._trackedMetrics.timeToFirstTokenMs !== undefined) {
this._ldClient.logger?.warn(
'Time to first token has already been tracked for this execution. Use createTracker() for a new execution.',
'Skipping trackTimeToFirstToken: time-to-first-token already recorded on this tracker. Call createTracker on the AI Config for a new run.',
);
return;
}
Expand Down Expand Up @@ -148,7 +148,7 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker {
trackFeedback(feedback: { kind: LDFeedbackKind }): void {
if (this._trackedMetrics.feedback !== undefined) {
this._ldClient.logger?.warn(
'Feedback has already been tracked for this execution. Use createTracker() for a new execution.',
'Skipping trackFeedback: feedback already recorded on this tracker. Call createTracker on the AI Config for a new run.',
);
return;
}
Expand All @@ -163,7 +163,7 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker {
trackSuccess(): void {
if (this._trackedMetrics.success !== undefined) {
this._ldClient.logger?.warn(
'Generation result has already been tracked for this execution. Use createTracker() for a new execution.',
'Skipping trackSuccess: success/error already recorded on this tracker. Call createTracker on the AI Config for a new run.',
);
return;
}
Expand All @@ -174,7 +174,7 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker {
trackError(): void {
if (this._trackedMetrics.success !== undefined) {
this._ldClient.logger?.warn(
'Generation result has already been tracked for this execution. Use createTracker() for a new execution.',
'Skipping trackError: success/error already recorded on this tracker. Call createTracker on the AI Config for a new run.',
);
return;
}
Expand Down Expand Up @@ -303,7 +303,7 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker {
trackTokens(tokens: LDTokenUsage): void {
if (this._trackedMetrics.tokens !== undefined) {
this._ldClient.logger?.warn(
'Token usage has already been tracked for this execution. Use createTracker() for a new execution.',
'Skipping trackTokens: token usage already recorded on this tracker. Call createTracker on the AI Config for a new run.',
);
return;
}
Expand Down
10 changes: 5 additions & 5 deletions packages/sdk/server-ai/src/LDGraphTrackerImpl.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ export class LDGraphTrackerImpl implements LDGraphTracker {
trackInvocationSuccess(): void {
if (this._summary.success !== undefined) {
this._ldClient.logger?.warn(
'LDGraphTracker: invocation success/failure already recorded for this run — dropping duplicate call.',
'Skipping trackInvocationSuccess: invocation result already recorded on this graph tracker. Call createTracker on the agent graph for a new run.',
);
return;
}
Expand All @@ -100,7 +100,7 @@ export class LDGraphTrackerImpl implements LDGraphTracker {
trackInvocationFailure(): void {
if (this._summary.success !== undefined) {
this._ldClient.logger?.warn(
'LDGraphTracker: invocation success/failure already recorded for this run — dropping duplicate call.',
'Skipping trackInvocationFailure: invocation result already recorded on this graph tracker. Call createTracker on the agent graph for a new run.',
);
return;
}
Expand All @@ -111,7 +111,7 @@ export class LDGraphTrackerImpl implements LDGraphTracker {
trackDuration(durationMs: number): void {
if (this._summary.durationMs !== undefined) {
this._ldClient.logger?.warn(
'LDGraphTracker: trackDuration already called for this run — dropping duplicate call.',
'Skipping trackDuration: duration already recorded on this graph tracker. Call createTracker on the agent graph for a new run.',
);
return;
}
Expand All @@ -127,7 +127,7 @@ export class LDGraphTrackerImpl implements LDGraphTracker {
trackTotalTokens(tokens: LDTokenUsage): void {
if (this._summary.tokens !== undefined) {
this._ldClient.logger?.warn(
'LDGraphTracker: trackTotalTokens already called for this run — dropping duplicate call.',
'Skipping trackTotalTokens: tokens already recorded on this graph tracker. Call createTracker on the agent graph for a new run.',
);
return;
}
Expand All @@ -143,7 +143,7 @@ export class LDGraphTrackerImpl implements LDGraphTracker {
trackPath(path: string[]): void {
if (this._summary.path !== undefined) {
this._ldClient.logger?.warn(
'LDGraphTracker: trackPath already called for this run — dropping duplicate call.',
'Skipping trackPath: path already recorded on this graph tracker. Call createTracker on the agent graph for a new run.',
);
return;
}
Expand Down
79 changes: 40 additions & 39 deletions packages/sdk/server-ai/src/api/LDAIClient.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,11 @@ export interface LDAIClient {
* the message content. The keys correspond to placeholders within the template, and the values
* are the corresponding replacements.
*
* @returns The AI `config`, customized `messages`, and a `tracker`. If the configuration cannot be accessed from
* LaunchDarkly, then the return value will include information from the `defaultValue`. The returned `tracker` can
* be used to track AI operation metrics (latency, token usage, etc.).
* @returns An {@link LDAICompletionConfig} with `enabled`, `model`, `provider`,
* `messages`, and a `createTracker()` factory. Call `createTracker()` on the
* returned config to obtain a tracker for each AI run. If the configuration
* cannot be accessed from LaunchDarkly, the return value will include
* information from the `defaultValue`.
*
* @example
* ```
Expand All @@ -49,27 +51,11 @@ export interface LDAIClient {
* provider: { name: 'openai' },
* };
*
* const result = completionConfig(key, context, defaultValue, variables);
* // Output:
* {
* enabled: true,
* config: {
* modelId: "gpt-4o",
* temperature: 0.2,
* maxTokens: 4096,
* userDefinedKey: "myValue",
* },
* messages: [
* {
* role: "system",
* content: "You are an amazing GPT."
* },
* {
* role: "user",
* content: "Explain how you're an amazing GPT."
* }
* ],
* tracker: ...
* const completionConfig = await client.completionConfig(key, context, defaultValue, variables);
* if (completionConfig.enabled) {
* const tracker = completionConfig.createTracker();
* // Use completionConfig.messages and completionConfig.model with your LLM,
* // then record metrics with tracker.trackSuccess(), tracker.trackTokens(), etc.
* }
* ```
*/
Expand All @@ -95,9 +81,11 @@ export interface LDAIClient {
* the instructions. The keys correspond to placeholders within the template, and the values
* are the corresponding replacements.
*
* @returns An AI agent with customized `instructions` and a `tracker`. If the configuration
* cannot be accessed from LaunchDarkly, then the return value will include information from the
* `defaultValue`. The returned `tracker` can be used to track AI operation metrics (latency, token usage, etc.).
* @returns An {@link LDAIAgentConfig} with customized `instructions`, `model`,
* `provider`, and a `createTracker()` factory. Call `createTracker()` on the
* returned config to obtain a tracker for each AI run. If the configuration
* cannot be accessed from LaunchDarkly, the return value will include
* information from the `defaultValue`.
*
* @example
* ```
Expand All @@ -111,8 +99,11 @@ export interface LDAIClient {
* instructions: 'You are a research assistant.',
* }, variables);
*
* const researchResult = agentConfig.instructions; // Interpolated instructions
* agentConfig.tracker.trackSuccess();
* if (agentConfig.enabled) {
* const tracker = agentConfig.createTracker();
* const researchResult = agentConfig.instructions; // Interpolated instructions
* tracker.trackSuccess();
* }
* ```
*/
agentConfig(
Expand All @@ -134,7 +125,10 @@ export interface LDAIClient {
* @param defaultValue Optional fallback when the configuration is not available from LaunchDarkly.
* When omitted or null, a disabled default is used.
* @param variables Optional variables for template interpolation in messages and instructions.
* @returns A promise that resolves to a tracked judge configuration.
* @returns A promise that resolves to an {@link LDAIJudgeConfig} with `enabled`,
* `model`, `provider`, `messages`, `evaluationMetricKey`, and a `createTracker()`
* factory. Call `createTracker()` on the returned config to obtain a tracker for
* each AI run.
*
* @example
* ```typescript
Expand All @@ -146,8 +140,11 @@ export interface LDAIClient {
* messages: [{ role: 'system', content: 'You are a relevance judge.' }]
* }, variables);
*
* const config = judgeConf.config; // Interpolated configuration
* judgeConf.tracker.trackSuccess();
* if (judgeConf.enabled) {
* const tracker = judgeConf.createTracker();
* // Use judgeConf.messages and judgeConf.model with your LLM,
* // then record metrics with tracker.trackSuccess(), tracker.trackJudgeResult(), etc.
* }
* ```
*/
judgeConfig(
Expand All @@ -167,10 +164,11 @@ export interface LDAIClient {
* current environment, user, or session. This context may influence how the configuration is
* processed or personalized.
*
* @returns A map of agent keys to their respective AI agents with customized `instructions` and `tracker`.
* If a configuration cannot be accessed from LaunchDarkly, then the return value will include information
* from the respective `defaultValue`. The returned `tracker` can be used to track AI operation metrics
* (latency, token usage, etc.).
* @returns A map of agent keys to their respective {@link LDAIAgentConfig}s,
* each with customized `instructions` and a `createTracker()` factory. Call
* `createTracker()` on a returned config to obtain a tracker for each AI run.
* If a configuration cannot be accessed from LaunchDarkly, the return value
* will include information from the respective `defaultValue`.
*
* @example
* ```
Expand Down Expand Up @@ -199,8 +197,11 @@ export interface LDAIClient {
* const context = {...};
*
* const configs = await client.agentConfigs(agentConfigsList, context);
* const researchResult = configs["research_agent"].instructions; // Interpolated instructions
* configs["research_agent"].tracker.trackSuccess();
* if (configs["research_agent"].enabled) {
* const tracker = configs["research_agent"].createTracker();
* const researchResult = configs["research_agent"].instructions; // Interpolated instructions
* tracker.trackSuccess();
* }
* ```
*/
agentConfigs<const T extends readonly LDAIAgentRequestConfig[]>(
Expand Down Expand Up @@ -316,7 +317,7 @@ export interface LDAIClient {
/**
* Reconstructs an AIConfigTracker from a resumption token string previously
* obtained from a tracker's `resumptionToken` property. Use this to associate
* deferred events (such as user feedback) with the original invocation's runId.
* deferred events (such as user feedback) with the original tracker's runId.
*
* @param token A URL-safe Base64-encoded resumption token string.
* @param context The evaluation context to use for subsequent track calls.
Expand Down
Loading
Loading