fix(analytics): Capture token usage and model name for Langfuse, LangSmith, and other providers (fixes #5763) (#5764)

* fix(analytics): capture token usage and model for Langfuse, LangSmith, and other providers

What changed
------------
- handler.ts: Extended onLLMEnd() to accept string | structured output. When
  structured output is passed, we now extract content, usageMetadata (input/
  output/total tokens), and responseMetadata (model name) and forward them
  to all analytics providers. Added usage/model to Langfuse generation.end(),
  LangSmith llm_output, and token attributes for Lunary, LangWatch, Arize,
  Phoenix, and Opik. Call langfuse.flushAsync() after generation.end() so
  updates are sent before the request completes.
- LLM.ts: Pass full output object from prepareOutputObject() to onLLMEnd
  instead of finalResponse string, so usage and model are available.
- Agent.ts: Same as LLM.ts — pass output object to onLLMEnd.
- ConditionAgent.ts: Build analyticsOutput with content, usageMetadata, and
  responseMetadata from the LLM response and pass to onLLMEnd.
- handler.test.ts: Added unit tests for the extraction logic (string vs
  object, token field normalization, model name sources, missing fields).
  OpenAIAssistant.ts call sites unchanged (Assistants API; no usage data).

Why
---
Fixes #5763. Analytics (Langfuse, LangSmith, etc.) were only receiving
plain text from onLLMEnd; usage_metadata and response_metadata from
AIMessage were dropped, so token counts and model names were missing in
dashboards and cost tracking.

Testing
-------
- pnpm build succeeds with no TypeScript errors.
- Manual: Flowise started, Agentflow with ChatOpenAI run; LangSmith and
  Langfuse both show token usage and model on the LLM generation.
- Backward compatible: call sites that pass a string (e.g. OpenAIAssistant)
  still work; onLLMEnd treats string as content-only.

Co-authored-by: Cursor <cursoragent@cursor.com>

* refactor(analytics): address PR review feedback for token usage handling

- LangSmith: Only include token_usage properties that have defined values
  to avoid passing undefined to the API
- Extract common OpenTelemetry span logic into _endOtelSpan helper method
  used by arize, phoenix, and opik providers

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(analytics): LangSmith cost tracking and flow name in traces

- LangSmith: set usage_metadata and ls_model_name/ls_provider on run extra.metadata
  so LangSmith can compute costs from token counts (compatible with langsmith 0.1.6
  which has no end(metadata) param). Infer ls_provider from model name.
- buildAgentflow: use chatflow.name as analytics trace/run name instead of
  hardcoded 'Agentflow' so LangSmith and Langfuse show the Flowise flow name.

Co-authored-by: Cursor <cursoragent@cursor.com>

* update handlers to include model and provider for analytics

* fix: normalize provider names in analytics handler to include 'amazon_bedrock'

---------

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Henry <hzj94@hotmail.com>

Author: 3 people

packages/components/nodes/agentflow/Agent/Agent.ts

@@ -686,6 +686,7 @@ class Agent_Agentflow implements INode {
             if (!model) {
                 throw new Error('Model is required')
             }
+            const modelName = modelConfig?.model ?? modelConfig?.modelName
 
             // Extract tools
             const tools = nodeData.inputs?.agentTools as ITool[]
@@ -1390,7 +1391,7 @@ class Agent_Agentflow implements INode {
 
             // End analytics tracking
             if (analyticHandlers && llmIds) {
-                await analyticHandlers.onLLMEnd(llmIds, finalResponse)
+                await analyticHandlers.onLLMEnd(llmIds, output, { model: modelName, provider: model })
             }
 
             // Send additional streaming events if needed

packages/components/nodes/agentflow/ConditionAgent/ConditionAgent.ts

@@ -260,6 +260,8 @@ class ConditionAgent_Agentflow implements INode {
             if (!model) {
                 throw new Error('Model is required')
             }
+            const modelName = modelConfig?.model ?? modelConfig?.modelName
+
             const conditionAgentInput = nodeData.inputs?.conditionAgentInput as string
             let input = conditionAgentInput || question
             const conditionAgentInstructions = nodeData.inputs?.conditionAgentInstructions as string
@@ -377,12 +379,20 @@ class ConditionAgent_Agentflow implements INode {
             const endTime = Date.now()
             const timeDelta = endTime - startTime
 
-            // End analytics tracking
+            // End analytics tracking (pass structured output with usage metadata)
             if (analyticHandlers && llmIds) {
-                await analyticHandlers.onLLMEnd(
-                    llmIds,
-                    typeof response.content === 'string' ? response.content : JSON.stringify(response.content)
-                )
+                const analyticsOutput: any = {
+                    content: typeof response.content === 'string' ? response.content : JSON.stringify(response.content)
+                }
+                // Include usage metadata if available
+                if (response.usage_metadata) {
+                    analyticsOutput.usageMetadata = response.usage_metadata
+                }
+                // Include response metadata (contains model name) if available
+                if (response.response_metadata) {
+                    analyticsOutput.responseMetadata = response.response_metadata
+                }
+                await analyticHandlers.onLLMEnd(llmIds, analyticsOutput, { model: modelName, provider: model })
             }
 
             let calledOutputName: string

packages/components/nodes/agentflow/LLM/LLM.ts

@@ -348,6 +348,7 @@ class LLM_Agentflow implements INode {
             if (!model) {
                 throw new Error('Model is required')
             }
+            const modelName = modelConfig?.model ?? modelConfig?.modelName
 
             // Extract memory and configuration options
             const enableMemory = nodeData.inputs?.llmEnableMemory as boolean
@@ -576,7 +577,7 @@ class LLM_Agentflow implements INode {
 
             // End analytics tracking
             if (analyticHandlers && llmIds) {
-                await analyticHandlers.onLLMEnd(llmIds, finalResponse)
+                await analyticHandlers.onLLMEnd(llmIds, output, { model: modelName, provider: model })
             }
 
             // Send additional streaming events if needed

packages/components/src/Interface.ts

@@ -458,7 +458,7 @@ export enum FollowUpPromptProvider {
 }
 
 export type FollowUpPromptProviderConfig = {
-    [key in FollowUpPromptProvider]: {
+    [_key in FollowUpPromptProvider]: {
         credentialId: string
         modelName: string
         baseUrl: string

packages/components/src/handler.test.ts

@@ -48,3 +48,286 @@ describe('URL Handling For Phoenix Tracer', () => {
         )
     })
 })
+
+/**
+ * Unit tests for onLLMEnd usage metadata extraction
+ *
+ * These tests verify the logic for extracting and formatting usage metadata
+ * from the onLLMEnd output parameter. Due to Jest configuration constraints
+ * with the complex OpenTelemetry and analytics dependencies, these tests are
+ * implemented as pure function tests that verify the extraction logic.
+ */
+describe('onLLMEnd Usage Metadata Extraction Logic', () => {
+    // Helper function that mirrors the extraction logic in handler.ts onLLMEnd (lines 1437-1465)
+    const extractOutputData = (output: string | Record<string, any>, model?: string) => {
+        let outputText: string
+        let usageMetadata: Record<string, any> | undefined
+        let modelName: string | undefined = model
+
+        if (typeof output === 'string') {
+            outputText = output
+        } else {
+            outputText = output.content ?? ''
+            usageMetadata = output.usageMetadata ?? output.usage_metadata
+            if (usageMetadata) {
+                usageMetadata = {
+                    input_tokens: usageMetadata.input_tokens ?? usageMetadata.prompt_tokens,
+                    output_tokens: usageMetadata.output_tokens ?? usageMetadata.completion_tokens,
+                    total_tokens: usageMetadata.total_tokens
+                }
+            }
+            const responseMetadata = output.responseMetadata ?? output.response_metadata
+            if (!model && responseMetadata) {
+                modelName = responseMetadata.model ?? responseMetadata.model_name ?? responseMetadata.modelId
+            }
+        }
+        return { outputText, usageMetadata, modelName }
+    }
+
+    // Helper to format for Langfuse
+    const formatForLangfuse = (usageMetadata: Record<string, any> | undefined) => {
+        if (!usageMetadata) return undefined
+        return {
+            promptTokens: usageMetadata.input_tokens,
+            completionTokens: usageMetadata.output_tokens,
+            totalTokens: usageMetadata.total_tokens
+        }
+    }
+
+    // Helper to format for LangSmith
+    const formatForLangSmith = (usageMetadata: Record<string, any> | undefined) => {
+        if (!usageMetadata) return undefined
+        return {
+            prompt_tokens: usageMetadata.input_tokens,
+            completion_tokens: usageMetadata.output_tokens,
+            total_tokens: usageMetadata.total_tokens
+        }
+    }
+
+    describe('backward compatibility with string input', () => {
+        it('should handle plain string output', () => {
+            const result = extractOutputData('Hello, world!')
+            expect(result.outputText).toBe('Hello, world!')
+            expect(result.usageMetadata).toBeUndefined()
+            expect(result.modelName).toBeUndefined()
+        })
+
+        it('should handle empty string', () => {
+            const result = extractOutputData('')
+            expect(result.outputText).toBe('')
+        })
+    })
+
+    describe('structured input with usage metadata', () => {
+        it('should extract usage metadata using LangChain field names (input_tokens/output_tokens)', () => {
+            const result = extractOutputData({
+                content: 'Test response',
+                usageMetadata: {
+                    input_tokens: 100,
+                    output_tokens: 50,
+                    total_tokens: 150
+                },
+                responseMetadata: {
+                    model: 'gpt-4'
+                }
+            })
+
+            expect(result.outputText).toBe('Test response')
+            expect(result.usageMetadata).toEqual({
+                input_tokens: 100,
+                output_tokens: 50,
+                total_tokens: 150
+            })
+            expect(result.modelName).toBe('gpt-4')
+        })
+
+        it('should handle OpenAI field names (prompt_tokens/completion_tokens)', () => {
+            const result = extractOutputData({
+                content: 'Test response',
+                usageMetadata: {
+                    prompt_tokens: 200,
+                    completion_tokens: 100,
+                    total_tokens: 300
+                }
+            })
+
+            // Should normalize to input_tokens/output_tokens
+            expect(result.usageMetadata).toEqual({
+                input_tokens: 200,
+                output_tokens: 100,
+                total_tokens: 300
+            })
+        })
+
+        it('should handle usage_metadata (snake_case) field name', () => {
+            const result = extractOutputData({
+                content: 'Test response',
+                usage_metadata: {
+                    input_tokens: 50,
+                    output_tokens: 25,
+                    total_tokens: 75
+                }
+            })
+
+            expect(result.usageMetadata).toEqual({
+                input_tokens: 50,
+                output_tokens: 25,
+                total_tokens: 75
+            })
+        })
+
+        it('should prefer usageMetadata over usage_metadata', () => {
+            const result = extractOutputData({
+                content: 'Test',
+                usageMetadata: { input_tokens: 100, output_tokens: 50, total_tokens: 150 },
+                usage_metadata: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }
+            })
+
+            expect(result.usageMetadata?.input_tokens).toBe(100)
+        })
+    })
+
+    describe('model name extraction', () => {
+        it('should extract model from responseMetadata.model', () => {
+            const result = extractOutputData({
+                content: 'Test',
+                responseMetadata: { model: 'gpt-4-turbo' }
+            })
+            expect(result.modelName).toBe('gpt-4-turbo')
+        })
+
+        it('should extract model from responseMetadata.model_name', () => {
+            const result = extractOutputData({
+                content: 'Test',
+                responseMetadata: { model_name: 'claude-3-opus' }
+            })
+            expect(result.modelName).toBe('claude-3-opus')
+        })
+
+        it('should extract model from responseMetadata.modelId', () => {
+            const result = extractOutputData({
+                content: 'Test',
+                responseMetadata: { modelId: 'anthropic.claude-v2' }
+            })
+            expect(result.modelName).toBe('anthropic.claude-v2')
+        })
+
+        it('should handle response_metadata (snake_case) field name', () => {
+            const result = extractOutputData({
+                content: 'Test',
+                response_metadata: { model: 'gpt-3.5-turbo' }
+            })
+            expect(result.modelName).toBe('gpt-3.5-turbo')
+        })
+
+        it('should prefer model over model_name over modelId', () => {
+            const result = extractOutputData({
+                content: 'Test',
+                responseMetadata: {
+                    model: 'preferred-model',
+                    model_name: 'secondary-model',
+                    modelId: 'tertiary-model'
+                }
+            })
+            expect(result.modelName).toBe('preferred-model')
+        })
+
+        it('should prefer explicit model param over responseMetadata', () => {
+            const result = extractOutputData(
+                {
+                    content: 'Test',
+                    responseMetadata: { model: 'from-response-metadata' }
+                },
+                'explicit-model-param'
+            )
+            expect(result.modelName).toBe('explicit-model-param')
+        })
+    })
+
+    describe('Langfuse format conversion', () => {
+        it('should format usage for Langfuse OpenAIUsage schema', () => {
+            const result = extractOutputData({
+                content: 'Test',
+                usageMetadata: { input_tokens: 100, output_tokens: 50, total_tokens: 150 }
+            })
+            const langfuseUsage = formatForLangfuse(result.usageMetadata)
+
+            expect(langfuseUsage).toEqual({
+                promptTokens: 100,
+                completionTokens: 50,
+                totalTokens: 150
+            })
+        })
+
+        it('should return undefined for missing usage', () => {
+            const result = extractOutputData({ content: 'Test' })
+            expect(formatForLangfuse(result.usageMetadata)).toBeUndefined()
+        })
+    })
+
+    describe('LangSmith format conversion', () => {
+        it('should format usage for LangSmith token_usage schema', () => {
+            const result = extractOutputData({
+                content: 'Test',
+                usageMetadata: { input_tokens: 100, output_tokens: 50, total_tokens: 150 }
+            })
+            const langSmithUsage = formatForLangSmith(result.usageMetadata)
+
+            expect(langSmithUsage).toEqual({
+                prompt_tokens: 100,
+                completion_tokens: 50,
+                total_tokens: 150
+            })
+        })
+    })
+
+    describe('missing fields handling', () => {
+        it('should handle structured output without usageMetadata', () => {
+            const result = extractOutputData({ content: 'Test response' })
+            expect(result.outputText).toBe('Test response')
+            expect(result.usageMetadata).toBeUndefined()
+            expect(result.modelName).toBeUndefined()
+        })
+
+        it('should handle structured output with only model, no usage', () => {
+            const result = extractOutputData({
+                content: 'Test response',
+                responseMetadata: { model: 'gpt-4' }
+            })
+            expect(result.usageMetadata).toBeUndefined()
+            expect(result.modelName).toBe('gpt-4')
+        })
+
+        it('should handle empty content', () => {
+            const result = extractOutputData({
+                content: '',
+                usageMetadata: { input_tokens: 10, output_tokens: 0, total_tokens: 10 }
+            })
+            expect(result.outputText).toBe('')
+            expect(result.usageMetadata).toEqual({
+                input_tokens: 10,
+                output_tokens: 0,
+                total_tokens: 10
+            })
+        })
+
+        it('should handle missing content field', () => {
+            const result = extractOutputData({
+                usageMetadata: { input_tokens: 10, output_tokens: 5, total_tokens: 15 }
+            })
+            expect(result.outputText).toBe('')
+        })
+
+        it('should handle undefined values in usage metadata', () => {
+            const result = extractOutputData({
+                content: 'Test',
+                usageMetadata: { input_tokens: 100 }
+            })
+            expect(result.usageMetadata).toEqual({
+                input_tokens: 100,
+                output_tokens: undefined,
+                total_tokens: undefined
+            })
+        })
+    })
+})