fix(llmobs): format AI SDK rich tool results

packages/dd-trace/src/llmobs/plugins/ai/util.js

@@ -13,6 +13,8 @@ const MODEL_METADATA_KEYS = new Set([
 const VERCEL_AI_TELEMETRY_METADATA_PREFIX = 'ai.telemetry.metadata.'
 const VERCEL_AI_MODEL_METADATA_PREFIX = 'gen_ai.request.'
 const VERCEL_AI_GENERATION_METADATA_PREFIX = 'ai.settings.'
+const UNPARSABLE_TOOL_RESULT = '[Unparsable Tool Result]'
+const UNSUPPORTED_TOOL_RESULT = '[Unsupported Tool Result]'
 
 /**
  * @typedef {import('../../../opentracing/span')} Span
@@ -26,8 +28,31 @@ const VERCEL_AI_GENERATION_METADATA_PREFIX = 'ai.settings.'
 
 /**
  * @typedef {{
- *   type: string,
- *   value?: unknown
+ *   type: 'text',
+ *   text: string
+ * } | {
+ *   type: 'media' | 'file',
+ *   mediaType: string
+ * } | {
+ *   type: 'file-data' | 'file-url' | 'file-id' | 'file-reference'
+ * } | {
+ *   type: 'image-data' | 'image-url' | 'image-file-id' | 'image-file-reference'
+ * } | {
+ *   type: 'custom'
+ * }} ToolCallContentPart
+ *
+ * @typedef {{
+ *   type: 'text' | 'error-text',
+ *   value: string
+ * } | {
+ *   type: 'json' | 'error-json',
+ *   value: unknown
+ * } | {
+ *   type: 'content',
+ *   value: ToolCallContentPart[]
+ * } | {
+ *   type: 'execution-denied',
+ *   reason?: string
  * }} ToolCallOutput
  *
  * @typedef {{ output?: ToolCallOutput, result?: unknown } & Record<string, unknown>} ToolCallResultContent
@@ -304,32 +329,89 @@ function getToolNameFromTags (tags) {
 }
 
 /**
- * Get the content of a tool call result.
- * Version 5 of the ai sdk sets this tag as `content.output`, with a `
- * @param {{ output?: { type: string, value: unknown }, result?: unknown }} content
+ * @param {unknown} value
+ * @returns {string}
+ */
+function stringifyToolCallResult (value) {
+  return JSON.stringify(value) ?? UNPARSABLE_TOOL_RESULT
+}
+
+/**
+ * @param {ToolCallContentPart[]} value
+ * @returns {string}
+ */
+function formatToolCallContent (value) {
+  if (!Array.isArray(value)) return UNPARSABLE_TOOL_RESULT
+
+  let result = ''
+  for (const part of value) {
+    if (typeof part !== 'object' || part === null) return UNPARSABLE_TOOL_RESULT
+
+    const { type } = part
+    if (type === 'text') {
+      if (typeof part.text !== 'string') return UNPARSABLE_TOOL_RESULT
+      result += part.text
+    } else if (type === 'media' || type === 'file') {
+      const { mediaType } = part
+      if (typeof mediaType !== 'string') return UNPARSABLE_TOOL_RESULT
+      result += mediaType === 'image' || mediaType.startsWith('image/') ? '[Image]' : '[File]'
+    } else if (
+      type === 'file-data' ||
+      type === 'file-url' ||
+      type === 'file-id' ||
+      type === 'file-reference'
+    ) {
+      result += '[File]'
+    } else if (
+      type === 'image-data' ||
+      type === 'image-url' ||
+      type === 'image-file-id' ||
+      type === 'image-file-reference'
+    ) {
+      result += '[Image]'
+    } else if (type === 'custom') {
+      result += '[Custom Content]'
+    } else {
+      return UNPARSABLE_TOOL_RESULT
+    }
+  }
+
+  return result
+}
+
+/**
+ * @param {ToolCallResultContent | null | undefined} content
  * @returns {string}
  */
 function getToolCallResultContent (content) {
-  const { output, result } = content
-  if (output) {
-    if (output.type === 'text') {
-      return output.value
-    } else if (output.type === 'json') {
-      return JSON.stringify(output.value)
+  try {
+    if (typeof content !== 'object' || content === null) return UNPARSABLE_TOOL_RESULT
+
+    const { output, result } = content
+    if (output !== undefined) {
+      if (typeof output !== 'object' || output === null) return UNPARSABLE_TOOL_RESULT
+
+      const { type, value } = output
+      if (type === 'text' || type === 'error-text') {
+        return typeof value === 'string' ? value : UNPARSABLE_TOOL_RESULT
+      } else if (type === 'json' || type === 'error-json') {
+        return stringifyToolCallResult(value)
+      } else if (type === 'content') {
+        return formatToolCallContent(value)
+      } else if (type === 'execution-denied') {
+        const { reason } = output
+        if (reason === undefined) return '[Tool Execution Denied]'
+        return typeof reason === 'string' ? reason : UNPARSABLE_TOOL_RESULT
+      }
+      return UNPARSABLE_TOOL_RESULT
     }
-    return '[Unparsable Tool Result]'
-  } else if (result) {
-    if (typeof result === 'string') {
-      return result
+    if (result !== undefined) {
+      return typeof result === 'string' ? result : stringifyToolCallResult(result)
     }
 
-    try {
-      return JSON.stringify(result)
-    } catch {
-      return '[Unparsable Tool Result]'
-    }
-  } else {
-    return '[Unsupported Tool Result]'
+    return UNSUPPORTED_TOOL_RESULT
+  } catch {
+    return UNPARSABLE_TOOL_RESULT
   }
 }
 

packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0361f19b.json

@@ -0,0 +1,50 @@
+{
+  "request": {
+    "method": "POST",
+    "url": "https://api.openai.com/v1/chat/completions",
+    "headers": {
+      "Connection": "keep-alive",
+      "Content-Type": "application/json",
+      "User-Agent": "ai/5.0.218 ai-sdk/provider-utils/3.0.30 runtime/node.js/24",
+      "Accept": "*/*",
+      "Accept-Language": "*",
+      "sec-fetch-mode": "cors",
+      "Accept-Encoding": "gzip, deflate",
+      "Content-Length": "558"
+    },
+    "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"content\":\"{\\\"message\\\":\\\"tool failure\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}"
+  },
+  "response": {
+    "status": {
+      "code": 200,
+      "message": "OK"
+    },
+    "headers": {
+      "Date": "Wed, 22 Jul 2026 20:13:31 GMT",
+      "Content-Type": "application/json",
+      "Transfer-Encoding": "chunked",
+      "Connection": "keep-alive",
+      "CF-Ray": "a1f51b39dc8af82f-EWR",
+      "CF-Cache-Status": "DYNAMIC",
+      "Server": "cloudflare",
+      "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload",
+      "X-Content-Type-Options": "nosniff",
+      "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray",
+      "openai-processing-ms": "614",
+      "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd",
+      "openai-version": "2020-10-01",
+      "x-openai-proxy-wasm": "v0.1",
+      "x-ratelimit-limit-requests": "30000",
+      "x-ratelimit-limit-tokens": "150000000",
+      "x-ratelimit-remaining-requests": "29999",
+      "x-ratelimit-remaining-tokens": "149999985",
+      "x-ratelimit-reset-requests": "2ms",
+      "x-ratelimit-reset-tokens": "0s",
+      "x-request-id": "req_6e1a2998fd994324bdc2bd3ae10e3b19",
+      "set-cookie": "__cf_bm=YKu653J0zCeRmStBop1XaLs6fJWkrbz81oF6AMvk3Uw-1784751210.5337121-1.0.1.1-CC0wup.gI4qlYGFdD0aSjt7HqBGyfpOZU62zc.bTIq.SkmUw2hyT.n2jL4HcHnhFgzZkt_GuAacYFbzdji6fn5oXH3pLCYraOvYzfg.eU7eV9mkLC8Ngk2XuJmj889Bn; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:31 GMT",
+      "Content-Encoding": "gzip",
+      "alt-svc": "h3=\":443\"; ma=86400"
+    },
+    "body": "{\n  \"id\": \"chatcmpl-E4XbmnhhTOv8VuwXC9YKsgbUKoh53\",\n  \"object\": \"chat.completion\",\n  \"created\": 1784751210,\n  \"model\": \"gpt-4o-mini-2024-07-18\",\n  \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\": \"assistant\",\n        \"content\": \"It seems there was an error when trying to run the test tool. The tool returned a failure message. Let me know if there's anything else you'd like to do or explore!\",\n        \"refusal\": null,\n        \"annotations\": []\n      },\n      \"logprobs\": null,\n      \"finish_reason\": \"stop\"\n    }\n  ],\n  \"usage\": {\n    \"prompt_tokens\": 68,\n    \"completion_tokens\": 36,\n    \"total_tokens\": 104,\n    \"prompt_tokens_details\": {\n      \"cached_tokens\": 0,\n      \"audio_tokens\": 0\n    },\n    \"completion_tokens_details\": {\n      \"reasoning_tokens\": 0,\n      \"audio_tokens\": 0,\n      \"accepted_prediction_tokens\": 0,\n      \"rejected_prediction_tokens\": 0\n    }\n  },\n  \"service_tier\": \"default\",\n  \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n"
+  }
+}

packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0529368f.json

@@ -0,0 +1,50 @@
+{
+  "request": {
+    "method": "POST",
+    "url": "https://api.openai.com/v1/chat/completions",
+    "headers": {
+      "Connection": "keep-alive",
+      "Content-Type": "application/json",
+      "User-Agent": "ai/7.0.31 ai-sdk/provider-utils/5.0.0 runtime/node.js/24",
+      "Accept": "*/*",
+      "Accept-Language": "*",
+      "sec-fetch-mode": "cors",
+      "Accept-Encoding": "gzip, deflate",
+      "Content-Length": "1163"
+    },
+    "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"[{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"before\\\"},{\\\"type\\\":\\\"file\\\",\\\"mediaType\\\":\\\"image/png\\\",\\\"data\\\":{\\\"type\\\":\\\"data\\\",\\\"data\\\":\\\"cHJpdmF0ZS1pbWFnZS1kYXRh\\\"}},{\\\"type\\\":\\\"file\\\",\\\"mediaType\\\":\\\"application/pdf\\\",\\\"data\\\":{\\\"type\\\":\\\"data\\\",\\\"data\\\":\\\"cHJpdmF0ZS1maWxlLWRhdGE=\\\"}},{\\\"type\\\":\\\"file\\\",\\\"mediaType\\\":\\\"application/pdf\\\",\\\"data\\\":{\\\"type\\\":\\\"reference\\\",\\\"reference\\\":{\\\"test\\\":\\\"private-file-id\\\"}}},{\\\"type\\\":\\\"file\\\",\\\"mediaType\\\":\\\"image/png\\\",\\\"data\\\":{\\\"type\\\":\\\"reference\\\",\\\"reference\\\":{\\\"test\\\":\\\"private-image-id\\\"}}},{\\\"type\\\":\\\"custom\\\",\\\"providerOptions\\\":{\\\"secret\\\":\\\"provider-data\\\"}},{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"after\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}"
+  },
+  "response": {
+    "status": {
+      "code": 200,
+      "message": "OK"
+    },
+    "headers": {
+      "Date": "Wed, 22 Jul 2026 20:14:35 GMT",
+      "Content-Type": "application/json",
+      "Transfer-Encoding": "chunked",
+      "Connection": "keep-alive",
+      "CF-Ray": "a1f51cc4eed4558f-EWR",
+      "CF-Cache-Status": "DYNAMIC",
+      "Server": "cloudflare",
+      "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload",
+      "X-Content-Type-Options": "nosniff",
+      "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray",
+      "openai-processing-ms": "1456",
+      "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd",
+      "openai-version": "2020-10-01",
+      "x-openai-proxy-wasm": "v0.1",
+      "x-ratelimit-limit-requests": "30000",
+      "x-ratelimit-limit-tokens": "150000000",
+      "x-ratelimit-remaining-requests": "29999",
+      "x-ratelimit-remaining-tokens": "149999855",
+      "x-ratelimit-reset-requests": "2ms",
+      "x-ratelimit-reset-tokens": "0s",
+      "x-request-id": "req_ad352507330141ffb9fa2c32e9beed74",
+      "set-cookie": "__cf_bm=cGrX9T7WZTNYAiDLRBsji0Wm88HYpSxoSeEs52rseXc-1784751273.7446923-1.0.1.1-e6qf06EukNFlCF4bAl0AV.W7a2KMVtVg.nTGe.LgdDn7ewb7cNi0sdsC54KAAJnIy56ttxZwjxM0FxvFfezWKPlGaNjdAmhfHonkKb7h0JO3RcyhR73i8jkpIOKVPoOI; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:44:35 GMT",
+      "Content-Encoding": "gzip",
+      "alt-svc": "h3=\":443\"; ma=86400"
+    },
+    "body": "{\n  \"id\": \"chatcmpl-E4Xcovu6Kem6n8VYf3IRKOSorwOzI\",\n  \"object\": \"chat.completion\",\n  \"created\": 1784751274,\n  \"model\": \"gpt-4o-mini-2024-07-18\",\n  \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\": \"assistant\",\n        \"content\": \"The test tool has been executed successfully, and here are the results:\\n\\n1. **Text Output**: \\\"before\\\"\\n2. **File Outputs**:\\n   - **Image (PNG)**: Data of the image (base64 encoded).\\n   - **PDF**: Data of the PDF (base64 encoded).\\n   - **PDF (reference)**: Reference to a private file ID.\\n   - **Image (reference)**: Reference to a private image ID.\\n3. **Custom Output**: Contains provider options with a secret (\\\"provider-data\\\").\\n4. **Text Output**: \\\"after\\\"\\n\\nIf you need any specific detail or further processing for any of these outputs, please let me know!\",\n        \"refusal\": null,\n        \"annotations\": []\n      },\n      \"logprobs\": null,\n      \"finish_reason\": \"stop\"\n    }\n  ],\n  \"usage\": {\n    \"prompt_tokens\": 217,\n    \"completion_tokens\": 141,\n    \"total_tokens\": 358,\n    \"prompt_tokens_details\": {\n      \"cached_tokens\": 0,\n      \"audio_tokens\": 0\n    },\n    \"completion_tokens_details\": {\n      \"reasoning_tokens\": 0,\n      \"audio_tokens\": 0,\n      \"accepted_prediction_tokens\": 0,\n      \"rejected_prediction_tokens\": 0\n    }\n  },\n  \"service_tier\": \"default\",\n  \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n"
+  }
+}

packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_09caa127.json

@@ -0,0 +1,50 @@
+{
+  "request": {
+    "method": "POST",
+    "url": "https://api.openai.com/v1/chat/completions",
+    "headers": {
+      "Connection": "keep-alive",
+      "Content-Type": "application/json",
+      "User-Agent": "ai/5.0.218",
+      "Accept": "*/*",
+      "Accept-Language": "*",
+      "sec-fetch-mode": "cors",
+      "Accept-Encoding": "gzip, deflate",
+      "Content-Length": "536"
+    },
+    "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"content\":\"[{\\\"type\\\":\\\"unknown\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}"
+  },
+  "response": {
+    "status": {
+      "code": 200,
+      "message": "OK"
+    },
+    "headers": {
+      "Date": "Wed, 22 Jul 2026 20:13:25 GMT",
+      "Content-Type": "application/json",
+      "Transfer-Encoding": "chunked",
+      "Connection": "keep-alive",
+      "CF-Ray": "a1f51b1539540f69-EWR",
+      "CF-Cache-Status": "DYNAMIC",
+      "Server": "cloudflare",
+      "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload",
+      "X-Content-Type-Options": "nosniff",
+      "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray",
+      "openai-processing-ms": "491",
+      "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd",
+      "openai-version": "2020-10-01",
+      "x-openai-proxy-wasm": "v0.1",
+      "x-ratelimit-limit-requests": "30000",
+      "x-ratelimit-limit-tokens": "150000000",
+      "x-ratelimit-remaining-requests": "29999",
+      "x-ratelimit-remaining-tokens": "149999987",
+      "x-ratelimit-reset-requests": "2ms",
+      "x-ratelimit-reset-tokens": "0s",
+      "x-request-id": "req_10b5047ada3644d5b49cf674f4178817",
+      "set-cookie": "__cf_bm=pCM4br.NJaCS_q3lJ7RgSCtgULi9q_CZARrA7qujqpc-1784751204.681613-1.0.1.1-VSYB8tzdJcDE3E_cIGT6QHL1tA562__HsypCMqKrb5oP4JcmoaVzv3SdrFA0IRXCCieehEuUPvkQBSYb86mBDuvBpY.DhrdtG8aTAXcuQI80Uo4VhhfFsObJ0OI8maKN; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:25 GMT",
+      "Content-Encoding": "gzip",
+      "alt-svc": "h3=\":443\"; ma=86400"
+    },
+    "body": "{\n  \"id\": \"chatcmpl-E4XbgbuVSTdLedI8u3Qyiff75AQvo\",\n  \"object\": \"chat.completion\",\n  \"created\": 1784751204,\n  \"model\": \"gpt-4o-mini-2024-07-18\",\n  \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\": \"assistant\",\n        \"content\": \"The test tool ran and returned a result of type \\\"unknown.\\\" If you need further assistance or specific information, please let me know!\",\n        \"refusal\": null,\n        \"annotations\": []\n      },\n      \"logprobs\": null,\n      \"finish_reason\": \"stop\"\n    }\n  ],\n  \"usage\": {\n    \"prompt_tokens\": 69,\n    \"completion_tokens\": 28,\n    \"total_tokens\": 97,\n    \"prompt_tokens_details\": {\n      \"cached_tokens\": 0,\n      \"audio_tokens\": 0\n    },\n    \"completion_tokens_details\": {\n      \"reasoning_tokens\": 0,\n      \"audio_tokens\": 0,\n      \"accepted_prediction_tokens\": 0,\n      \"rejected_prediction_tokens\": 0\n    }\n  },\n  \"service_tier\": \"default\",\n  \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n"
+  }
+}