diff --git a/packages/dd-trace/src/llmobs/plugins/ai/util.js b/packages/dd-trace/src/llmobs/plugins/ai/util.js index 0909431e9a..8b629dab50 100644 --- a/packages/dd-trace/src/llmobs/plugins/ai/util.js +++ b/packages/dd-trace/src/llmobs/plugins/ai/util.js @@ -13,6 +13,8 @@ const MODEL_METADATA_KEYS = new Set([ const VERCEL_AI_TELEMETRY_METADATA_PREFIX = 'ai.telemetry.metadata.' const VERCEL_AI_MODEL_METADATA_PREFIX = 'gen_ai.request.' const VERCEL_AI_GENERATION_METADATA_PREFIX = 'ai.settings.' +const UNPARSABLE_TOOL_RESULT = '[Unparsable Tool Result]' +const UNSUPPORTED_TOOL_RESULT = '[Unsupported Tool Result]' /** * @typedef {import('../../../opentracing/span')} Span @@ -26,8 +28,31 @@ const VERCEL_AI_GENERATION_METADATA_PREFIX = 'ai.settings.' /** * @typedef {{ - * type: string, - * value?: unknown + * type: 'text', + * text: string + * } | { + * type: 'media' | 'file', + * mediaType: string + * } | { + * type: 'file-data' | 'file-url' | 'file-id' | 'file-reference' + * } | { + * type: 'image-data' | 'image-url' | 'image-file-id' | 'image-file-reference' + * } | { + * type: 'custom' + * }} ToolCallContentPart + * + * @typedef {{ + * type: 'text' | 'error-text', + * value: string + * } | { + * type: 'json' | 'error-json', + * value: unknown + * } | { + * type: 'content', + * value: ToolCallContentPart[] + * } | { + * type: 'execution-denied', + * reason?: string * }} ToolCallOutput * * @typedef {{ output?: ToolCallOutput, result?: unknown } & Record} ToolCallResultContent @@ -304,32 +329,89 @@ function getToolNameFromTags (tags) { } /** - * Get the content of a tool call result. - * Version 5 of the ai sdk sets this tag as `content.output`, with a ` - * @param {{ output?: { type: string, value: unknown }, result?: unknown }} content + * @param {unknown} value + * @returns {string} + */ +function stringifyToolCallResult (value) { + return JSON.stringify(value) ?? UNPARSABLE_TOOL_RESULT +} + +/** + * @param {ToolCallContentPart[]} value + * @returns {string} + */ +function formatToolCallContent (value) { + if (!Array.isArray(value)) return UNPARSABLE_TOOL_RESULT + + let result = '' + for (const part of value) { + if (typeof part !== 'object' || part === null) return UNPARSABLE_TOOL_RESULT + + const { type } = part + if (type === 'text') { + if (typeof part.text !== 'string') return UNPARSABLE_TOOL_RESULT + result += part.text + } else if (type === 'media' || type === 'file') { + const { mediaType } = part + if (typeof mediaType !== 'string') return UNPARSABLE_TOOL_RESULT + result += mediaType === 'image' || mediaType.startsWith('image/') ? '[Image]' : '[File]' + } else if ( + type === 'file-data' || + type === 'file-url' || + type === 'file-id' || + type === 'file-reference' + ) { + result += '[File]' + } else if ( + type === 'image-data' || + type === 'image-url' || + type === 'image-file-id' || + type === 'image-file-reference' + ) { + result += '[Image]' + } else if (type === 'custom') { + result += '[Custom Content]' + } else { + return UNPARSABLE_TOOL_RESULT + } + } + + return result +} + +/** + * @param {ToolCallResultContent | null | undefined} content * @returns {string} */ function getToolCallResultContent (content) { - const { output, result } = content - if (output) { - if (output.type === 'text') { - return output.value - } else if (output.type === 'json') { - return JSON.stringify(output.value) + try { + if (typeof content !== 'object' || content === null) return UNPARSABLE_TOOL_RESULT + + const { output, result } = content + if (output !== undefined) { + if (typeof output !== 'object' || output === null) return UNPARSABLE_TOOL_RESULT + + const { type, value } = output + if (type === 'text' || type === 'error-text') { + return typeof value === 'string' ? value : UNPARSABLE_TOOL_RESULT + } else if (type === 'json' || type === 'error-json') { + return stringifyToolCallResult(value) + } else if (type === 'content') { + return formatToolCallContent(value) + } else if (type === 'execution-denied') { + const { reason } = output + if (reason === undefined) return '[Tool Execution Denied]' + return typeof reason === 'string' ? reason : UNPARSABLE_TOOL_RESULT + } + return UNPARSABLE_TOOL_RESULT } - return '[Unparsable Tool Result]' - } else if (result) { - if (typeof result === 'string') { - return result + if (result !== undefined) { + return typeof result === 'string' ? result : stringifyToolCallResult(result) } - try { - return JSON.stringify(result) - } catch { - return '[Unparsable Tool Result]' - } - } else { - return '[Unsupported Tool Result]' + return UNSUPPORTED_TOOL_RESULT + } catch { + return UNPARSABLE_TOOL_RESULT } } diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0361f19b.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0361f19b.json new file mode 100644 index 0000000000..d34da23252 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0361f19b.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218 ai-sdk/provider-utils/3.0.30 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "558" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"content\":\"{\\\"message\\\":\\\"tool failure\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:31 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b39dc8af82f-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "614", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999985", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_6e1a2998fd994324bdc2bd3ae10e3b19", + "set-cookie": "__cf_bm=YKu653J0zCeRmStBop1XaLs6fJWkrbz81oF6AMvk3Uw-1784751210.5337121-1.0.1.1-CC0wup.gI4qlYGFdD0aSjt7HqBGyfpOZU62zc.bTIq.SkmUw2hyT.n2jL4HcHnhFgzZkt_GuAacYFbzdji6fn5oXH3pLCYraOvYzfg.eU7eV9mkLC8Ngk2XuJmj889Bn; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:31 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbmnhhTOv8VuwXC9YKsgbUKoh53\",\n \"object\": \"chat.completion\",\n \"created\": 1784751210,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems there was an error when trying to run the test tool. The tool returned a failure message. Let me know if there's anything else you'd like to do or explore!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 68,\n \"completion_tokens\": 36,\n \"total_tokens\": 104,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0529368f.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0529368f.json new file mode 100644 index 0000000000..aa06581a43 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0529368f.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/7.0.31 ai-sdk/provider-utils/5.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "1163" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"[{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"before\\\"},{\\\"type\\\":\\\"file\\\",\\\"mediaType\\\":\\\"image/png\\\",\\\"data\\\":{\\\"type\\\":\\\"data\\\",\\\"data\\\":\\\"cHJpdmF0ZS1pbWFnZS1kYXRh\\\"}},{\\\"type\\\":\\\"file\\\",\\\"mediaType\\\":\\\"application/pdf\\\",\\\"data\\\":{\\\"type\\\":\\\"data\\\",\\\"data\\\":\\\"cHJpdmF0ZS1maWxlLWRhdGE=\\\"}},{\\\"type\\\":\\\"file\\\",\\\"mediaType\\\":\\\"application/pdf\\\",\\\"data\\\":{\\\"type\\\":\\\"reference\\\",\\\"reference\\\":{\\\"test\\\":\\\"private-file-id\\\"}}},{\\\"type\\\":\\\"file\\\",\\\"mediaType\\\":\\\"image/png\\\",\\\"data\\\":{\\\"type\\\":\\\"reference\\\",\\\"reference\\\":{\\\"test\\\":\\\"private-image-id\\\"}}},{\\\"type\\\":\\\"custom\\\",\\\"providerOptions\\\":{\\\"secret\\\":\\\"provider-data\\\"}},{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"after\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:14:35 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51cc4eed4558f-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "1456", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999855", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_ad352507330141ffb9fa2c32e9beed74", + "set-cookie": "__cf_bm=cGrX9T7WZTNYAiDLRBsji0Wm88HYpSxoSeEs52rseXc-1784751273.7446923-1.0.1.1-e6qf06EukNFlCF4bAl0AV.W7a2KMVtVg.nTGe.LgdDn7ewb7cNi0sdsC54KAAJnIy56ttxZwjxM0FxvFfezWKPlGaNjdAmhfHonkKb7h0JO3RcyhR73i8jkpIOKVPoOI; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:44:35 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xcovu6Kem6n8VYf3IRKOSorwOzI\",\n \"object\": \"chat.completion\",\n \"created\": 1784751274,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has been executed successfully, and here are the results:\\n\\n1. **Text Output**: \\\"before\\\"\\n2. **File Outputs**:\\n - **Image (PNG)**: Data of the image (base64 encoded).\\n - **PDF**: Data of the PDF (base64 encoded).\\n - **PDF (reference)**: Reference to a private file ID.\\n - **Image (reference)**: Reference to a private image ID.\\n3. **Custom Output**: Contains provider options with a secret (\\\"provider-data\\\").\\n4. **Text Output**: \\\"after\\\"\\n\\nIf you need any specific detail or further processing for any of these outputs, please let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 217,\n \"completion_tokens\": 141,\n \"total_tokens\": 358,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_09caa127.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_09caa127.json new file mode 100644 index 0000000000..3ae952aacc --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_09caa127.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "536" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"content\":\"[{\\\"type\\\":\\\"unknown\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:25 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b1539540f69-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "491", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_10b5047ada3644d5b49cf674f4178817", + "set-cookie": "__cf_bm=pCM4br.NJaCS_q3lJ7RgSCtgULi9q_CZARrA7qujqpc-1784751204.681613-1.0.1.1-VSYB8tzdJcDE3E_cIGT6QHL1tA562__HsypCMqKrb5oP4JcmoaVzv3SdrFA0IRXCCieehEuUPvkQBSYb86mBDuvBpY.DhrdtG8aTAXcuQI80Uo4VhhfFsObJ0OI8maKN; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:25 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbgbuVSTdLedI8u3Qyiff75AQvo\",\n \"object\": \"chat.completion\",\n \"created\": 1784751204,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool ran and returned a result of type \\\"unknown.\\\" If you need further assistance or specific information, please let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 69,\n \"completion_tokens\": 28,\n \"total_tokens\": 97,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0b6f9297.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0b6f9297.json new file mode 100644 index 0000000000..1bd2406986 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0b6f9297.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218 ai-sdk/provider-utils/3.0.30 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "1139" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"content\":\"[{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"before\\\"},{\\\"type\\\":\\\"media\\\",\\\"mediaType\\\":\\\"image/png\\\",\\\"data\\\":\\\"private-image-data\\\"},{\\\"type\\\":\\\"media\\\",\\\"mediaType\\\":\\\"application/pdf\\\",\\\"data\\\":\\\"private-file-data\\\"},{\\\"type\\\":\\\"file-data\\\",\\\"mediaType\\\":\\\"application/pdf\\\",\\\"data\\\":\\\"private-file-data\\\"},{\\\"type\\\":\\\"file-id\\\",\\\"fileId\\\":\\\"private-file-id\\\"},{\\\"type\\\":\\\"image-data\\\",\\\"mediaType\\\":\\\"image/png\\\",\\\"data\\\":\\\"private-image-data\\\"},{\\\"type\\\":\\\"image-file-id\\\",\\\"fileId\\\":\\\"private-image-id\\\"},{\\\"type\\\":\\\"custom\\\",\\\"providerOptions\\\":{\\\"secret\\\":\\\"provider-data\\\"}},{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"after\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:28 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b1cbdc5fef2-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "2094", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999862", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_d45d4c4aa6154079acb9ba3e5d1fd06d", + "set-cookie": "__cf_bm=mpYZZ398HYHntWHmVHBRj2nndk7WrFG6l6J.2wlnq5U-1784751205.878469-1.0.1.1-yChyq3Mi4NV7U0iWTlk6tuCCRNLe.JT_WNOh7msyOjs32yBJIXSCVCaDM8hKkuF3WwsIHx0f2F3nGsQscd6FdYlyJqJ6TZu0GO6Cs6QsH5vaSbELNSmy0ZpKdCCiuas9; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:28 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xbi4NNoi2wHb7RGCRHZ6lyDUfPn\",\n \"object\": \"chat.completion\",\n \"created\": 1784751206,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has been executed, and here are the results:\\n\\n1. **Text**: \\\"before\\\"\\n2. **Media (Image)**: PNG data (private)\\n3. **Media (PDF)**: PDF data (private)\\n4. **File Data (PDF)**: PDF file data (private)\\n5. **File ID**: private-file-id\\n6. **Image Data**: PNG image data (private)\\n7. **Image File ID**: private-image-id\\n8. **Custom**: Provider options with secret (private data)\\n9. **Text**: \\\"after\\\"\\n\\nIf you need any specific information or further actions based on these results, let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 185,\n \"completion_tokens\": 141,\n \"total_tokens\": 326,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_1a817464.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_1a817464.json new file mode 100644 index 0000000000..a5abc273a0 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_1a817464.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "518" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"content\":\"result\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:21 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51afe0b7af834-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "554", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_297c199134694dc6b763c81286de676d", + "set-cookie": "__cf_bm=EYO0B1S_ra4XD6097FmJ9hnxcw2kOpyVoUtVyYWdRtE-1784751200.9636052-1.0.1.1-PDvZhHjL7z4yX4t_hJzFBy6_bmfSQJHCd5XgnbWs28HC8aQO3VmKUbgug.8lvjJZoiGvcKEDOpdtTTR8HWYqu6MeIle.3rJHDSID6BF7Bt2jVlOm0dN2o2npmz6sKmeu; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:21 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xbdvwo8PsNteG6VniMS5JCkLUcl\",\n \"object\": \"chat.completion\",\n \"created\": 1784751201,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has successfully run and returned the result. If you need more specific information or further assistance, please let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 63,\n \"completion_tokens\": 27,\n \"total_tokens\": 90,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_1b92db1c.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_1b92db1c.json new file mode 100644 index 0000000000..fb67c9926d --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_1b92db1c.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.40 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "519" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"result\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:43 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b89ed1cb637-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "492", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_d96e1a04d0214f86b2294bab0859eb59", + "set-cookie": "__cf_bm=xtSVSvtJlPb5QbLDYGLYell_RpKExImPcS3MQ6tR0nA-1784751223.3429987-1.0.1.1-MXuBKd3ASlYJSiVi2rqPcAiwjStik13EWl2oX7a3sgvKbUyhy.GhEkn_gD5QN9pVTui56EQ.EG3U65jpfK1Vw3REOLQAi2UwmYMhbIyJ2sQqrQDPcNVHlhaq9r9oWvpR; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:43 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbzdYCDs2V5nzdUAbS4WQHKokjW\",\n \"object\": \"chat.completion\",\n \"created\": 1784751223,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has successfully run and returned the result. Would you like to know more about it or need assistance with anything else?\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 63,\n \"completion_tokens\": 27,\n \"total_tokens\": 90,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_20b36358.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_20b36358.json new file mode 100644 index 0000000000..42cf30bbf4 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_20b36358.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "523" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"tool failure\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:37 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b61cd81fbfb-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "599", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_62204cf23b1340339458766ff2b175be", + "set-cookie": "__cf_bm=XbJVsCo9UlomqUW0e38lKhKkqFkWv82L9KxI7cLFQ5M-1784751216.9298835-1.0.1.1-qEWhQpQSb7AAftdgJKmrXIui80Aa999HaE8iRL4pne53AwGyKFpCPoB4sCQR92bWjpgDBCyL4ZIel656Pr1c9EOSQLUmXfYwTbtCWBsqEO2nAiK_XXlYKSqqyZITzCIS; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:37 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbthUpMY96YvaKt2oKEvStCch6t\",\n \"object\": \"chat.completion\",\n \"created\": 1784751217,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems that there was a failure when trying to run the test tool. If you need assistance with something specific, please let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 64,\n \"completion_tokens\": 29,\n \"total_tokens\": 93,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_20e896cf.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_20e896cf.json new file mode 100644 index 0000000000..f4b4052c91 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_20e896cf.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "User-Agent": "node", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "276" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"temperature\":0,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"0\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:15 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51ad79af143d6-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "482", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999995", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_c6e1d1a71b1448c6a01a4789b68b96fd", + "set-cookie": "__cf_bm=Ku3aEuPaXwCuI._sg76.n2alVtPLJYj4dlULIw2dMIA-1784751194.8122158-1.0.1.1-GYu5iJGI26HhHwDQat1mTAET6_PHCXDmeTC6q7wh7uiy3224ETgLFTbSh0.tp4eyEYmmn5lQ9OvJrZ7QVxMwyV2mfqDLZDHhm.aYn6pmDo71ijhjD5rSEIkfG15Gk18d; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:15 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbXfTLgre8oEblfCmgvGvJmrwSK\",\n \"object\": \"chat.completion\",\n \"created\": 1784751195,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_q604yF7U7NC9ME4wbnaOfqHD\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"0\",\n \"arguments\": \"{}\"\n }\n }\n ],\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 44,\n \"completion_tokens\": 10,\n \"total_tokens\": 54,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_176e862412\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_22e8e00d.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_22e8e00d.json new file mode 100644 index 0000000000..a9a277d531 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_22e8e00d.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "1131" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"[{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"before\\\"},{\\\"type\\\":\\\"image-data\\\",\\\"data\\\":\\\"private-image-data\\\",\\\"mediaType\\\":\\\"image/png\\\"},{\\\"type\\\":\\\"file-data\\\",\\\"data\\\":\\\"private-file-data\\\",\\\"mediaType\\\":\\\"application/pdf\\\"},{\\\"type\\\":\\\"file-data\\\",\\\"mediaType\\\":\\\"application/pdf\\\",\\\"data\\\":\\\"private-file-data\\\"},{\\\"type\\\":\\\"file-id\\\",\\\"fileId\\\":\\\"private-file-id\\\"},{\\\"type\\\":\\\"image-data\\\",\\\"mediaType\\\":\\\"image/png\\\",\\\"data\\\":\\\"private-image-data\\\"},{\\\"type\\\":\\\"image-file-id\\\",\\\"fileId\\\":\\\"private-image-id\\\"},{\\\"type\\\":\\\"custom\\\",\\\"providerOptions\\\":{\\\"secret\\\":\\\"provider-data\\\"}},{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"after\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:35 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b4efda87611-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "1528", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999857", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_6bfb162699a848d9866dd8025748bcb2", + "set-cookie": "__cf_bm=xPGEY4GN3FLXmexCdEmwnF8lLQG_9063crRRgsXMKQg-1784751213.9207683-1.0.1.1-J_pMwqB_w_z_zChtxbRVc0cR1cb7u7BNQx4LDW74aaWvM1EuSs6W45DW.h0GjbSmseJeIVHlr5S_v.aBH8yrqzxtGyPy6mfMD2eKnLwhCefkMVxAqZOnI55_309C_p2t; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:35 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbqSj8WZ9GUxOt1AzXeAaDnD7Jk\",\n \"object\": \"chat.completion\",\n \"created\": 1784751214,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has been run and returned the following data:\\n\\n1. **Text**: \\\"before\\\"\\n2. **Image Data**: (private image data in PNG format)\\n3. **File Data**: (private file data in PDF format)\\n4. **File Data**: (another private file data in PDF format)\\n5. **File ID**: (private file ID)\\n6. **Image Data**: (private image data in PNG format)\\n7. **Image File ID**: (private image ID)\\n8. **Custom Data**: (provider options with secret data)\\n9. **Text**: \\\"after\\\"\\n\\nPlease let me know if you need any further assistance or analysis on this data!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 187,\n \"completion_tokens\": 147,\n \"total_tokens\": 334,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_23de8c14.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_23de8c14.json new file mode 100644 index 0000000000..869f550838 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_23de8c14.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.40 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "515" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"[]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:46 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b9c7cabda80-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "398", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_1b719827a70c46469531724e5c92d261", + "set-cookie": "__cf_bm=ByDsdIfHw1XGpPR_2g1BO26huZa0wQQZzN6xMHppM0w-1784751226.3170166-1.0.1.1-d0.lEGbk3Zy3VE6zBQYA0GHhgEJWip4ZJHcQypX960JqoVfbVjLpvshLjvSR8B.s.TJ1kgMYkmTnTa0VfMpdaDmM4ok7cXjx4d_obJvkJXqrxqTYdjGNhb4TRlyjlJIk; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:46 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xc2u32wFzClxUrJkOjwYApahNLs\",\n \"object\": \"chat.completion\",\n \"created\": 1784751226,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has run successfully, and there are no issues to report.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 63,\n \"completion_tokens\": 16,\n \"total_tokens\": 79,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_24e03d01.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_24e03d01.json new file mode 100644 index 0000000000..ca2518f555 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_24e03d01.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.40 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "543" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"{\\\"message\\\":\\\"tool failure\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:46 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b97c80860e6-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "600", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999985", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_be1349635cb04a1a854ac7d2515cbf7f", + "set-cookie": "__cf_bm=7Koc3j5iC4BcjlIQhENAOsyBS2iNntf9XE1nyY_ZOtM-1784751225.5662088-1.0.1.1-qPt.UxSZ.o_Ks1_enlxV5CKVG2EtDMBDhLxUuXFLE3w7xsyHA6Q._rlao1AJ6Uyp4pwGZvnbyJxgj0GjqTKn10KTx2ZC2V5RXqQPG3HqBWAXf6Skvv3QWWz_D2zb5djD; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:46 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xc1Id8CsCI59aHQEnY5aamMJFzo\",\n \"object\": \"chat.completion\",\n \"created\": 1784751225,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"There was a failure while attempting to run the test tool. If you need assistance with something specific, please let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 68,\n \"completion_tokens\": 26,\n \"total_tokens\": 94,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_2f30d5b0.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_2f30d5b0.json new file mode 100644 index 0000000000..f3fbf2df3a --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_2f30d5b0.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.40 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "530" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"Approval rejected\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:47 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b9ffcc2b12a-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "671", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_bb636c69889f421ebeb27db9c20067c4", + "set-cookie": "__cf_bm=ZMhjVUg40xaXkK7Cqo9uwGDTJnyo1LcFvmtnDVi5bMs-1784751226.8730886-1.0.1.1-x2.ptX1ozk34O9cFNEdm0YXBxxOQ47GrxlaNBvx8tf2xjueKr3zMXrYDVdjRqVn1PwM47L_FnWv_IgQfeDy2nVx4QYbz20H5ZHqXoAGYol9XeQ5zNRoMjJeFuJQd8A5Q; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:47 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xc3RJysabFeBmjZucvAdNS6kdQE\",\n \"object\": \"chat.completion\",\n \"created\": 1784751227,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems that the test tool execution was rejected. If you have any other requests or need assistance with something else, feel free to let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 64,\n \"completion_tokens\": 31,\n \"total_tokens\": 95,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_3c8e8f64.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_3c8e8f64.json new file mode 100644 index 0000000000..5bfe90d8ad --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_3c8e8f64.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218 ai-sdk/provider-utils/3.0.30 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "296" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:25 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b1978ddf2f9-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "378", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999992", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_0ccc5fed4b9f4e7e88c81c76f1b33ed1", + "set-cookie": "__cf_bm=P8FC.2zKl3RVBZGhTwmqssabz0cIdEBELaAxuYX3uRA-1784751205.3579347-1.0.1.1-ReLQDpggzHiZCntDPzvO289r8PWy0oaMEUSkcd13GZoM._XZeAOLoo.JmQCJI7o6Ias6YbXC0xxXLACnGfSzIhJquECbUDun_hE8MN7u_ofsW6evPTK0zCB48I0CVWFE; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:25 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xbhydjr9JkTAeQCGNskVrKmoMxS\",\n \"object\": \"chat.completion\",\n \"created\": 1784751205,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_6CIsJhzx7yJcZ0igevQZacnB\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"testTool\",\n \"arguments\": \"{}\"\n }\n }\n ],\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 44,\n \"completion_tokens\": 10,\n \"total_tokens\": 54,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_3de7cf9f.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_3de7cf9f.json new file mode 100644 index 0000000000..c67724b7f8 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_3de7cf9f.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "513" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"[]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:39 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b6c4c187a02-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "581", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999992", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_356ce2f3a65149ca9e12813b43d03da9", + "set-cookie": "__cf_bm=MjMCAMrPAiw1pcdDz5brZgZUYKvB4Tv6.x6Uj38QCao-1784751218.6057847-1.0.1.1-it15KMe0DnApYaPT9LgOBWsTNvFVV0owZH093vw6Jf1Qg3O..SqNkcZQhVvV1lcK6RaeYC.BtuO41Znr9Ho4k9WuN3DXrvDKeUrfXwPgQkIpEO1qY17OzVaNxYI9VFw0; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:39 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbuoPTqtuuY0RuzbvtkqYb5q0K0\",\n \"object\": \"chat.completion\",\n \"created\": 1784751218,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has run successfully, and it returned an empty result.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 63,\n \"completion_tokens\": 15,\n \"total_tokens\": 78,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_40f5975e.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_40f5975e.json new file mode 100644 index 0000000000..8429611f39 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_40f5975e.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "User-Agent": "node", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "500" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"temperature\":0,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_q604yF7U7NC9ME4wbnaOfqHD\",\"type\":\"function\",\"function\":{\"name\":\"0\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_q604yF7U7NC9ME4wbnaOfqHD\",\"content\":\"0\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"0\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:16 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51ae1d8708095-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "461", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999992", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_2470a02e38174c75b819395c57aaaf6a", + "set-cookie": "__cf_bm=n2qmzFo1ic2HmXH25168IMs56l1C_iwCTiYN6G8hrUA-1784751196.455854-1.0.1.1-6JHpCmlq3tkIalUWnP2vvcss2rledbWSoeIAsPQqjCxTmgkbMOH63srqISRSV4V612V711w1nr0s1jB1jV_XLlIHn80WD86zmBkEU6Y5qFsFr5lhy8Dm5tk0ARhWl2eP; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:16 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbY2oNI714Ubagbuih8ZiXneI9R\",\n \"object\": \"chat.completion\",\n \"created\": 1784751196,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has been successfully run, and the result is 0.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 62,\n \"completion_tokens\": 16,\n \"total_tokens\": 78,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_176e862412\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_444a857f.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_444a857f.json new file mode 100644 index 0000000000..9be787c2c2 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_444a857f.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "527" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"content\":\"{\\\"answer\\\":42}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:22 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b0269040cfb-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "376", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_41280cee84014927be0ecd99e14a6a83", + "set-cookie": "__cf_bm=I3raLAOPtBvp526rJcA_3wRkO.kpmGZ6CL4Elnlqn2M-1784751201.6612327-1.0.1.1-VEQoEQ5iIg2P2lRX1kbAErbb5zkoQ3_kPq92Sg7kUrZBUT02uAo4PiBqRat4_RXRsfyxzGAHM25s0t.K8eH90C..2Qj6rQFj7EnGmYNYJCyMemiWM2tp7e9N2f.yRV_e; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:22 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbdXPkVqWQqtC7p6NSu1phnzj24\",\n \"object\": \"chat.completion\",\n \"created\": 1784751201,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool returned the answer: 42.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 67,\n \"completion_tokens\": 11,\n \"total_tokens\": 78,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_4ced261c.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_4ced261c.json new file mode 100644 index 0000000000..b580f9920f --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_4ced261c.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "524" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"content\":\"tool failure\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:23 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b0588e732b1-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "908", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_d5083fe955fd4043a5b0534fb2069b40", + "set-cookie": "__cf_bm=87tOuW20ywX4MQkQNojhTe2gaiROiJ5a49_taI7lq7w-1784751202.1638446-1.0.1.1-0wxodCi7JjJfT8MEIfkm8tGu_xjhrZZ0M1wB_SGG0ojupeE2e71VSM10r.d1G9UkT3yJEw60ExzluQR9mxODb9J.tBJVCaveJI.KYxCr786KLL8BTs1.PKXf7RZP9SUy; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:23 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xbeo6qekjtam5gjVrQpRswru9Y7\",\n \"object\": \"chat.completion\",\n \"created\": 1784751202,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems that there was a failure when attempting to run the test tool. If you have any further instructions or if there's something specific you'd like to know or try, please let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 64,\n \"completion_tokens\": 39,\n \"total_tokens\": 103,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_57593709.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_57593709.json new file mode 100644 index 0000000000..8c885189d8 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_57593709.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.40 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "1133" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"[{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"before\\\"},{\\\"type\\\":\\\"image-data\\\",\\\"data\\\":\\\"private-image-data\\\",\\\"mediaType\\\":\\\"image/png\\\"},{\\\"type\\\":\\\"file-data\\\",\\\"data\\\":\\\"private-file-data\\\",\\\"mediaType\\\":\\\"application/pdf\\\"},{\\\"type\\\":\\\"file-data\\\",\\\"mediaType\\\":\\\"application/pdf\\\",\\\"data\\\":\\\"private-file-data\\\"},{\\\"type\\\":\\\"file-id\\\",\\\"fileId\\\":\\\"private-file-id\\\"},{\\\"type\\\":\\\"image-data\\\",\\\"mediaType\\\":\\\"image/png\\\",\\\"data\\\":\\\"private-image-data\\\"},{\\\"type\\\":\\\"image-file-id\\\",\\\"fileId\\\":\\\"private-image-id\\\"},{\\\"type\\\":\\\"custom\\\",\\\"providerOptions\\\":{\\\"secret\\\":\\\"provider-data\\\"}},{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"after\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:43 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b7f79d67ca5-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "1487", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999860", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_f1074054fab048ac88b4f01d5461905f", + "set-cookie": "__cf_bm=qBNnt.uECmOeSxrkl_3ZCpnjm1b5zpZPzmRGakuGWao-1784751221.6794298-1.0.1.1-g8L6H0EzVOE8NZlhs8UZtRHdwNST_K2e6nHv1TpoMPoQm6rQU0CX5yz1rh1yRbwqP8ytaGX.zn__P1SA46l2uofMycarwBN8xNxAd2w3E.bQb6CA01.USW14X553L0ik; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:43 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbxdCpE8KaPfAWKuODz61iYL525\",\n \"object\": \"chat.completion\",\n \"created\": 1784751221,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has been executed successfully. Here are the results:\\n\\n1. **Text**: \\\"before\\\"\\n2. **Image Data**: (Image data in PNG format)\\n3. **File Data**: (PDF file data)\\n4. **File Data**: (PDF file data)\\n5. **File ID**: (ID for a private file)\\n6. **Image Data**: (Image data in PNG format)\\n7. **Image File ID**: (ID for a private image)\\n8. **Custom Data**: (Includes provider options with secret)\\n9. **Text**: \\\"after\\\"\\n\\nIf you need further assistance or specific details about these results, let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 187,\n \"completion_tokens\": 143,\n \"total_tokens\": 330,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_5bbf5257.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_5bbf5257.json new file mode 100644 index 0000000000..729d44bd6d --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_5bbf5257.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "533" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"Tool execution denied.\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:40 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b75aab46e53-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "613", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_41efd80ca3cf450fa53d5c9373208b56", + "set-cookie": "__cf_bm=c06xgYCI9gejrgqYPFFdMtI0iugtMMJuEmj7aU5ZoEI-1784751220.104493-1.0.1.1-FNXn_3lin7ooTB9jLgjJa7E.3PD6pIzhEePrv2cSI7lxt2Fux8JO7aLRDwVrXbHYLfq6gPC44D1.Ym6iyeI14HeOcF51Dhz.q6IKtje.u7k3Dg0pa_wO4VT0HiegoaGP; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:40 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbwIoMppg1fNk9LYCJ5vzlAEI1U\",\n \"object\": \"chat.completion\",\n \"created\": 1784751220,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"I'm unable to execute the test tool at the moment due to restrictions. If there's something specific you'd like to know or another way I can assist you, please let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 66,\n \"completion_tokens\": 36,\n \"total_tokens\": 102,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_613472dd.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_613472dd.json new file mode 100644 index 0000000000..90f81aeef2 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_613472dd.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.40 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "540" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"Tool call execution denied.\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:48 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51ba4f91a7a9c-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "759", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999982", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_2569847e1d604d0890b6347393dae874", + "set-cookie": "__cf_bm=LsvM5mbIaOt2YS1B8GrlhEHd3RqBW6vVyPMdfxdZtZ0-1784751227.674948-1.0.1.1-fUoG.BAcFseDn.lYorJI2CkEdcmIkh.w75BuOOQxL0LcTcG.UWkdC09r.1IUfvE_pCInHsb70c.10wg9edHJHAY3ln4TH_V0sEb6kQbyHbhZMv3edwYmvT7cp9e_yqtM; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:48 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xc4CcfUre2asSIdEvmgXTMA1U6i\",\n \"object\": \"chat.completion\",\n \"created\": 1784751228,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems that I'm unable to run the test tool due to restrictions on execution. If you have any specific questions or need assistance with something else, feel free to ask!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 67,\n \"completion_tokens\": 35,\n \"total_tokens\": 102,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6163bd35.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6163bd35.json new file mode 100644 index 0000000000..1a84e837ff --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6163bd35.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "528" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"Approval rejected\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:40 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b715d929630-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "542", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_a0d7b60d7d0a493aaf3a19034667aa94", + "set-cookie": "__cf_bm=_NYAnQPr3oW2fhOZVXzJKqvDsdEj0L1moRqkKeVCCQM-1784751219.4145305-1.0.1.1-LhLGdqC6yXufp5DV1JSmK9SSBI6RwFlD2yuHZmRXR4u9Hsv_aOBQT6j53T9eIp_2n7yobRa6QEsZR.J2hapgtb3UAQDTsGvU40vZqd5.yHJucjusOCBJ.REbkddDdwhe; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:40 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbvgjRmNStygKS6QZNDgTbc1a4F\",\n \"object\": \"chat.completion\",\n \"created\": 1784751219,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool's execution resulted in an \\\"Approval rejected\\\" response. If you have any specific tasks or questions, feel free to let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 64,\n \"completion_tokens\": 31,\n \"total_tokens\": 95,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_62af4eb0.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_62af4eb0.json new file mode 100644 index 0000000000..89529873f0 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_62af4eb0.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218 ai-sdk/provider-utils/3.0.30 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "540" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"content\":\"tool failure\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:30 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b31dfe60d00-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "855", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_3c6be94d02a648babf9be23026a73253", + "set-cookie": "__cf_bm=RSNfG4jxmlpdX_2x2cJH04sZp4P4cKAeq0WVfgnXnJ4-1784751209.2534914-1.0.1.1-XNgapA16TOPv7Co7aQAGNf1EIINoV3ln81KcXjuewOfagBDkzh8Bnq4Cwx97hNx4ap7AI2UcqC6JPWWc5yF5yFWqsGMUTNjVFXFyUPlg_.AzhfdiQ6Q3BVWNbvblTScb; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:30 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xblgq0buhNTMYsMbpdKrTAXSiT1\",\n \"object\": \"chat.completion\",\n \"created\": 1784751209,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems that there was a failure when attempting to run the test tool. If you have any specific details or requests you'd like to share, I can try to assist further!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 64,\n \"completion_tokens\": 36,\n \"total_tokens\": 100,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6c05f6ae.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6c05f6ae.json new file mode 100644 index 0000000000..658596e45a --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6c05f6ae.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.40 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "525" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"tool failure\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:45 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b914beac4d1-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "867", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_5a7e00cd2f2f45cbaf9851aada34f451", + "set-cookie": "__cf_bm=Sj89kb3nTZkdSSOpgbhUJMytnotJKfvUO51BNDL5tB4-1784751224.5265749-1.0.1.1-7fsO2Monifz3jKTodw6WftFK7abHP_9g6FTuUj9.7WLAaXwz7fPoI2X.Gd6v1YxU_Q0ZohLllIvRWBhJ07OVvpCzR3OmoegYBLsgzNa9ZW0Ytyk.VPduE0WDkcHM4BNR; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:45 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xc0c8asWv7MM02JyzyvedPJ7GnP\",\n \"object\": \"chat.completion\",\n \"created\": 1784751224,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems there was an issue running the test tool. Would you like me to assist you with something else or try running the tool again?\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 64,\n \"completion_tokens\": 29,\n \"total_tokens\": 93,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6c485410.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6c485410.json new file mode 100644 index 0000000000..d5dae7247f --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6c485410.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "281" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:33 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b4b1bc23448-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "479", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999995", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_a07f4223ae4b42c98b1cd5cdc1664c2a", + "set-cookie": "__cf_bm=X07fg6iyVC2ZLgESW1t0V7csLl35TvtL2PWLZA.4U6c-1784751213.2972581-1.0.1.1-tT8FPQjXicdyMvB1lnnqyiurUc9Nl9zf8WSKFxQXsfLKmO7HYczcGYj7O2bjdiOy4sgDefDMHo.Xz0_MHPwgkm962RVsE6g.zf0e4QmDTHulFr3Y3zRaPrNX0ZmqUrK.; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:33 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xbp9eavrDkG1EdR47AqltNCElO1\",\n \"object\": \"chat.completion\",\n \"created\": 1784751213,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_ivr6y32ed6IgA8Df696Hq5TV\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"testTool\",\n \"arguments\": \"{}\"\n }\n }\n ],\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 44,\n \"completion_tokens\": 10,\n \"total_tokens\": 54,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_9cf39adf.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_9cf39adf.json new file mode 100644 index 0000000000..b917d0e515 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_9cf39adf.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "542" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"content\":\"{\\\"message\\\":\\\"tool failure\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:23 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b0c7890da48-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "424", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999985", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_a7579857c8d24373ac4229d38b385d4f", + "set-cookie": "__cf_bm=Y4.HBtnweGax8e9yelwxTiy9rl0CHGHP3xu2RUh0oH0-1784751203.271045-1.0.1.1-AuxB.AXslvF2t0sLXxTeBPUAMi.NwNa.K5ZOi993x2cBascPVNFDyB6YorIpY1VMDGeEHd6LeT93O5hYq9zw13VZriZ0BC6gfrUlokgr5OdJ8GyClUrWbL2ud6_PxAjl; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:23 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xbfibyy4npm8fAryRGiHyfgvv4R\",\n \"object\": \"chat.completion\",\n \"created\": 1784751203,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool encountered a failure. How can I assist you further?\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 68,\n \"completion_tokens\": 15,\n \"total_tokens\": 83,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a17b7d89.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a17b7d89.json new file mode 100644 index 0000000000..14bc786368 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a17b7d89.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "1123" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"content\":\"[{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"before\\\"},{\\\"type\\\":\\\"media\\\",\\\"mediaType\\\":\\\"image/png\\\",\\\"data\\\":\\\"private-image-data\\\"},{\\\"type\\\":\\\"media\\\",\\\"mediaType\\\":\\\"application/pdf\\\",\\\"data\\\":\\\"private-file-data\\\"},{\\\"type\\\":\\\"file-data\\\",\\\"mediaType\\\":\\\"application/pdf\\\",\\\"data\\\":\\\"private-file-data\\\"},{\\\"type\\\":\\\"file-id\\\",\\\"fileId\\\":\\\"private-file-id\\\"},{\\\"type\\\":\\\"image-data\\\",\\\"mediaType\\\":\\\"image/png\\\",\\\"data\\\":\\\"private-image-data\\\"},{\\\"type\\\":\\\"image-file-id\\\",\\\"fileId\\\":\\\"private-image-id\\\"},{\\\"type\\\":\\\"custom\\\",\\\"providerOptions\\\":{\\\"secret\\\":\\\"provider-data\\\"}},{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"after\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:20 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51af3bf244b05-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "1511", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999860", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_6249f88c64014cd1bf47344a3dfca024", + "set-cookie": "__cf_bm=qGUDyxGppIZmuRO79s_KGLMtRZPapFL_BNuUCio3b1Q-1784751199.314885-1.0.1.1-Nsmo7BiuF_SOSS3BX_3bpjsCRXCBmFQazwL4CeWZUZoVRb4F8doxj5atXcTpUCjVhpQ9WTsHqvJTpJMXQ7cYZCTA1Hnf7opb4F4yZdRqXCWICi.SoVdD1ddEPv2m1p19; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:20 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbbdhlckyPXklTDOMjygf5LIxVi\",\n \"object\": \"chat.completion\",\n \"created\": 1784751199,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has run successfully, and here are the results:\\n\\n- **Text Output**: \\\"before\\\"\\n- **Media Outputs**:\\n - Image (PNG): Data is available but private.\\n - PDF: Data is available but private (two representations).\\n- **File Information**:\\n - PDF: File ID is private.\\n - Image: File ID is private.\\n- **Custom Output**: Provider options contain secret data.\\n- **Text Output**: \\\"after\\\"\\n\\nIf you need further analysis or specific actions based on this output, let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 185,\n \"completion_tokens\": 116,\n \"total_tokens\": 301,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a233b9ea.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a233b9ea.json new file mode 100644 index 0000000000..64a11f012f --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a233b9ea.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "535" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"[{\\\"type\\\":\\\"unknown\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:41 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b7a89efa6af-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "627", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999985", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_1c254192d8574605a36257e5109ae566", + "set-cookie": "__cf_bm=iuia7OhoWEZY7lSpmY6ZX10b55NGLhg_h3D.o3R654s-1784751220.8863275-1.0.1.1-sIr2iezgv6mAAkOrkYblVkpFQmz2OrwKl1AwoF5GtN9SplUBnaXevlbxwQfAfo0iXreFr.0essqyop6pNaoh65_.bI5Q5tIw_L0y6xcVqbbbxMT5go1MTsq8efZ9i10k; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:41 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xbxmnk84PLXoT7bplxy4JnLdkEr\",\n \"object\": \"chat.completion\",\n \"created\": 1784751221,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems that the test tool returned an unexpected result with the type \\\"unknown.\\\" If you need further assistance or would like to run another test, please let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 69,\n \"completion_tokens\": 35,\n \"total_tokens\": 104,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a509302d.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a509302d.json new file mode 100644 index 0000000000..b0b5ed0a7f --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a509302d.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "User-Agent": "node", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "504" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"temperature\":0,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_q604yF7U7NC9ME4wbnaOfqHD\",\"type\":\"function\",\"function\":{\"name\":\"0\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_q604yF7U7NC9ME4wbnaOfqHD\",\"content\":\"false\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"0\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:17 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51ae5e8df4693-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "396", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_cfa8bea4c1cd425992005d7ca09be2a1", + "set-cookie": "__cf_bm=LZ5xXOVoNKgcA90JYCkxSYWJJQqbM5wFOOszP.hru8I-1784751197.1020737-1.0.1.1-GX.74RyXaRUI4VBIzYuZTr_PZO_wCM7kuk4jN6aQjNtgYdfq7a_NoEawxdWBgYFISBdfGRM1zuli7NAcksgqFbz8gXzarSEpT8QjHiclSHo9Fdh78kcwTPW1VyKBFAux; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:17 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbZv01VAKoxJX8NxHdbefVtekvS\",\n \"object\": \"chat.completion\",\n \"created\": 1784751197,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has returned a result of `false`.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 62,\n \"completion_tokens\": 12,\n \"total_tokens\": 74,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_176e862412\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a51ba16a.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a51ba16a.json new file mode 100644 index 0000000000..9915e6de50 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a51ba16a.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218 ai-sdk/provider-utils/3.0.30 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "534" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"content\":\"result\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:28 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b2adf54c413-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "437", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999992", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_9fb1a586cd984a3bba19f65c7443aa8b", + "set-cookie": "__cf_bm=EzEjU1HhKiY9hreJQIATKJOpxMBCsEWC9MIfLn0PAKs-1784751208.135568-1.0.1.1-FgY.uv1fBU_GPmVGsFatD1VksQKR4D5zSjU_Is2YkBsDULOYVEUumNVLbx85ad3E9ln0792xvYQKBjK8CJAbnM1ybwKKmeNN1UY21wlQlXEyRINCF0heI0RIMGaWnpMf; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:28 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbkTxii73qYvsuL9lFBmp6HdM76\",\n \"object\": \"chat.completion\",\n \"created\": 1784751208,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has been run successfully, and the result is \\\"result.\\\"\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 63,\n \"completion_tokens\": 16,\n \"total_tokens\": 79,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ad68426d.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ad68426d.json new file mode 100644 index 0000000000..7c43375f2c --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ad68426d.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "541" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"{\\\"message\\\":\\\"tool failure\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:38 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b670cf65594-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "697", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999985", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_2d93713c6e7e457993f81c8971c3712e", + "set-cookie": "__cf_bm=phxPS1gE0FdWZIT5AxLS3E.rOnWEG19zmZ87RDuz3rE-1784751217.7661781-1.0.1.1-LsTzBkm4m8szg_0iHXr6srtsMYfiLTm5VXaYKn75_y_26RMAMX0UgKODFSfSEVkrXkkawyiAawt_CKAMoM4hkMmKqR31BlbSHsm32KEgiKOKbGJxYPiVdpN1rx9bQ6Se; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:38 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbtiCYiG8SaNXcVlBmTC23WSqIJ\",\n \"object\": \"chat.completion\",\n \"created\": 1784751217,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems that there was a failure when attempting to run the test tool. If you have any other requests or need assistance with something else, feel free to let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 68,\n \"completion_tokens\": 36,\n \"total_tokens\": 104,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_adfc828f.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_adfc828f.json new file mode 100644 index 0000000000..90b0511309 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_adfc828f.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.40 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "528" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"{\\\"answer\\\":42}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:44 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b8dfbc627c6-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "398", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_4ea1b722c19340a1995a17f2d80cc3bb", + "set-cookie": "__cf_bm=cekRPRKQ_EWZLWb4CbVyU6RarEqf759OA2VOYApCXRU-1784751223.9933553-1.0.1.1-nAFttum.0veky8iCizQlrG33.6wP8upoTOwM8DDL6ZHQYjhtezjGGn3g9xpUKKnpuXn5D25ys.k0rnvNgu3MfzpEvRGBBK8vXVn8VcOEYmtmMKVQzh9oZ0kWkVowzdED; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:44 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xc0DmliNZMDOO5rEVnjgCRTcHt3\",\n \"object\": \"chat.completion\",\n \"created\": 1784751224,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool returned the answer: 42.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 67,\n \"completion_tokens\": 11,\n \"total_tokens\": 78,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_b9510f9a.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_b9510f9a.json new file mode 100644 index 0000000000..3404d04741 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_b9510f9a.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "517" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"result\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:36 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b597b952067-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "596", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_18679b6262724719a768fef0ebb89228", + "set-cookie": "__cf_bm=zu53laSLyfyXEspdvsbqK3ndkzvrALN0red4uvlVkN4-1784751215.6010828-1.0.1.1-WXI7vVSQN_M9HJ8tFkDVaGSByT7iTsUx.r9D2TnXtp8BstJ7_gQMejbOb4x1zi5WianpDu6u42JBK3yqGfVlMIe.pYaFDjkiRo6GUwpWKDADrPAKFq_NJmnpxd7kfHHJ; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:36 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbrMOv54SAbjYV5tKyZDKNUPXr3\",\n \"object\": \"chat.completion\",\n \"created\": 1784751215,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has been successfully executed, and the result is \\\"result.\\\" If you need any further assistance or details, feel free to ask!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 63,\n \"completion_tokens\": 30,\n \"total_tokens\": 93,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_c0ec5af0.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_c0ec5af0.json new file mode 100644 index 0000000000..f8b83b9012 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_c0ec5af0.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "User-Agent": "node", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "503" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"temperature\":0,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_q604yF7U7NC9ME4wbnaOfqHD\",\"type\":\"function\",\"function\":{\"name\":\"0\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_q604yF7U7NC9ME4wbnaOfqHD\",\"content\":\"\\\"\\\"\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"0\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:16 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51adcb93541f3-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "625", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999992", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_e0f1d5e68c3b4be38c6b3e94aec4ad3e", + "set-cookie": "__cf_bm=7Eq9Mj09AgZ_tgk3xJTfcSyR51ID6v9j3EXp_axKwE8-1784751195.634347-1.0.1.1-daQssUPtYhnf7GM8z7lRp0.YLX6fVXuRkDtWoM.1nZVDzNbPgdxKfiS62yxEiJ3BdXA88PFuDFvnFwcG9NEqzJs8pq951iEjVPOIR8BZthy.G2npliXatCQ3Zrs9pb7X; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:16 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbXwGkPpgqyjtl4sni9SgIATWa0\",\n \"object\": \"chat.completion\",\n \"created\": 1784751195,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has been run, and it returned an empty result. If you need further assistance or have specific tasks in mind, feel free to let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 62,\n \"completion_tokens\": 34,\n \"total_tokens\": 96,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_176e862412\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_d65abfba.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_d65abfba.json new file mode 100644 index 0000000000..96adef2604 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_d65abfba.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "514" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"content\":\"[]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:24 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b0fd81afe89-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "723", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_3ead4596161c4bd8bc82183a19ea2866", + "set-cookie": "__cf_bm=NRlAnE9YEbDhcqR0gXinhw8h_lT.8ex6MKXU7gNUm_8-1784751203.8204072-1.0.1.1-yjzQwdoKBYJV7Q3nBCG2tTtJ2I0vO7I3iamAUbGLY9WIAgab_k4vGYwt1btJUvNwmhfuTE.CpNRLRo_WPFGUU3VP.akQsU.HOSEJa2mVJ5WhQ4G_WVeswRgUU5iLsPTw; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:24 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbgqTe0w7dJJyedm7XiWemCAIl6\",\n \"object\": \"chat.completion\",\n \"created\": 1784751204,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has run successfully and returned an empty result. If you need further assistance or have any specific tasks in mind, feel free to let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 63,\n \"completion_tokens\": 33,\n \"total_tokens\": 96,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_e6f7f8ad.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_e6f7f8ad.json new file mode 100644 index 0000000000..656c01e39d --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_e6f7f8ad.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.40 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "537" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"[{\\\"type\\\":\\\"unknown\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:49 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51baacd1c7ca5-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "689", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999985", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_9a21210a1e65464da99121b5d8d972b6", + "set-cookie": "__cf_bm=YBa3kvvV1p5XfTA6nl6dsnxIABIa8R.GUpOLhne2ofE-1784751228.6086779-1.0.1.1-ybCNUCGoCLVntZI67Lsz811g3Qfln87yaR_VdxUvFB.All4ppikVUq70KVu3slQ1EdngxeXYVESMhojJsnhnJtMEFjn.5xddg3QGsyYRFdbkYPtCZuQ7jOBbmchW8eL2; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:49 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xc4mWZCtMXeQzHn8eoTMONG9Dx7\",\n \"object\": \"chat.completion\",\n \"created\": 1784751228,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool returned an unexpected result with a type categorized as \\\"unknown.\\\" If you have any specific tests or checks in mind that I should perform, please let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 69,\n \"completion_tokens\": 36,\n \"total_tokens\": 105,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ea7e2114.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ea7e2114.json new file mode 100644 index 0000000000..9fa0bf1595 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ea7e2114.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "User-Agent": "node", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "503" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"temperature\":0,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_q604yF7U7NC9ME4wbnaOfqHD\",\"type\":\"function\",\"function\":{\"name\":\"0\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_q604yF7U7NC9ME4wbnaOfqHD\",\"content\":\"null\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"0\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:18 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51ae9cad5114c-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "657", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_967755fe6eee4fe9bfab28750c6c09f1", + "set-cookie": "__cf_bm=I4UqtCrvps0v2ty3gz1xSFh26ZIuAX8Xiqicdshuh9U-1784751197.724774-1.0.1.1-4rAMdrQLz405QzLdUkF60cbJn3vbykmnHH2yQ9BpTTu56lHqeG8augxU92RGnVDo_IovNjgouF4xoPAX2Z7vV1ZHv4UpW95yGKdBQ6RJXu5qJ4O1m8ZHj_EI9wCSwxoy; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:18 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbZLyCelQ3nvFiq0xDDyY5Hi6pA\",\n \"object\": \"chat.completion\",\n \"created\": 1784751197,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has been run, and it returned a null result. If you have any specific tasks or questions, feel free to ask!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 62,\n \"completion_tokens\": 29,\n \"total_tokens\": 91,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_176e862412\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_f0546f43.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_f0546f43.json new file mode 100644 index 0000000000..352507b599 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_f0546f43.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218 ai-sdk/provider-utils/3.0.30 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "543" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"content\":\"{\\\"answer\\\":42}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:29 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b2ea8e54ba5-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "377", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_3da9f65def934135b9911152252e0b1b", + "set-cookie": "__cf_bm=nJBTYPDabmHNdR5pg59YNxg5nn6XjReam076HODmcFE-1784751208.743452-1.0.1.1-Ax7siAdtZeXIKEYNyCMsupm5A77y34C8uQJhqvEZhlComB_QQ8CUy2wkbvOXh4xRkE1ES7Qv9kX7noEGxPn99aQgXIC1O8M0XQFDx2nrrsxIGzDjDVo_NeBPC8TpXwNb; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:29 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbkThLDPnMm8iH15p8PmVQrvYcU\",\n \"object\": \"chat.completion\",\n \"created\": 1784751208,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool returned the answer: 42.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 67,\n \"completion_tokens\": 11,\n \"total_tokens\": 78,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_f1893fa3.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_f1893fa3.json new file mode 100644 index 0000000000..25df7907f5 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_f1893fa3.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218 ai-sdk/provider-utils/3.0.30 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "530" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"content\":\"[]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:32 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b3f2d6c10f3-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "617", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999992", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_4530a5faf6444944b5e4778c8b6f7b09", + "set-cookie": "__cf_bm=6_5U_Z3fUByPXyzymtL4UkNsg49jMyqQzHmnvTW4Pgg-1784751211.3817-1.0.1.1-PGuYa1WNodIwMWGVmB7uD_pMRmtu2hYKWNaoBJE7UsMfG5C0BZSWrGbTVy1riU6UcqqGMRlwNpnAk0q33b.UnPq2eVPu8fRVXBHlHw1fFAcHLzAvj5_rXjyHaviW5j8u; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:32 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbnMCi8Zy5Ez28pKEqhIZqRbe29\",\n \"object\": \"chat.completion\",\n \"created\": 1784751211,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool ran successfully, and there were no issues or errors reported. If you need further assistance or have any other requests, feel free to ask!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 63,\n \"completion_tokens\": 32,\n \"total_tokens\": 95,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_fe44e2dd.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_fe44e2dd.json new file mode 100644 index 0000000000..10566974a2 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_fe44e2dd.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "282" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:19 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51af01ce025d8-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "440", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999995", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_0e884b0f8b904434a4f06e24815083f3", + "set-cookie": "__cf_bm=PzlTWDz15ANmZ1wb9wHasUnOrz0eXJylzeuGWq2wfBQ-1784751198.7381263-1.0.1.1-I0FeBDkEOq1outWGxw0fDhoUwmLQ8I2MJD8uG_pcrCtdEBw01fkLJRJMQRjDSMy_No3NtuaUNlaPACyr5PTxxshRI7xQRokq4QVaR9D0ec1gHpSKxODsIucmzexzm38q; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:19 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbaJe2aGSrM21YQfqgww3Ro1fNn\",\n \"object\": \"chat.completion\",\n \"created\": 1784751198,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_tMh2IPARQABNSUlF7F8j20b1\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"testTool\",\n \"arguments\": \"{}\"\n }\n }\n ],\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 44,\n \"completion_tokens\": 10,\n \"total_tokens\": 54,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ff5c5e3d.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ff5c5e3d.json new file mode 100644 index 0000000000..eb48f5c669 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ff5c5e3d.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "526" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"{\\\"answer\\\":42}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:36 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b5e0fddde6d-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "372", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_7edc4616bfe840228c24cd1c74c5f10b", + "set-cookie": "__cf_bm=D3eloq1tHW95.9H0vqeworhMTqoy3kdMPxgyTOHZKxw-1784751216.3240464-1.0.1.1-yErURngg3hVKX_Xs6nbmfHEbytAbMJDVlsQlKH1PAYyPINtwZadmeiAh6G3NTdcZ6lFVLjVaM_kGBcqbN9ILsJWnjV4B1yWLr_mjtyKiCGM9Z7lQ0VTCptilduTsx4up; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:36 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbsY3n3vv3N5zbigam1ZOdubvrk\",\n \"object\": \"chat.completion\",\n \"created\": 1784751216,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool returned the answer: 42.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 67,\n \"completion_tokens\": 11,\n \"total_tokens\": 78,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ffbf4f37.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ffbf4f37.json new file mode 100644 index 0000000000..bf6a900748 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ffbf4f37.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218 ai-sdk/provider-utils/3.0.30 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "552" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"content\":\"[{\\\"type\\\":\\\"unknown\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:33 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b443caab547-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "739", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999985", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_300d60eb6b40492b87fb0e3eb0b99185", + "set-cookie": "__cf_bm=_ICJux.AGa1KeFSScSnFEOLM8tO9.tXDz0oymwkPC3g-1784751212.1932561-1.0.1.1-q_4cQNvCdSokThOa3dkaLUbTH9DAkAuubeyyHqY37h2LZEuawROmbjZn8KN8W3u2qYvSwm1mecjNNS3k5BFNICibwIPYx3Pa9JlbrJqisMaJVrhmGzgZ0YOw3gR_ozl4; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:33 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbowYSzLdthND0NmxbSOsvMxoPx\",\n \"object\": \"chat.completion\",\n \"created\": 1784751212,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems that the test tool has returned an unknown result. If you have specific questions or need assistance with something else, feel free to ask!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 69,\n \"completion_tokens\": 30,\n \"total_tokens\": 99,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js index a91863c5db..43ad731263 100644 --- a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js @@ -6,6 +6,7 @@ const semifies = require('semifies') const { useEnv } = require('../../../../../../integration-tests/helpers') const { withVersions } = require('../../../setup/mocha') const iastFilter = require('../../../../src/appsec/iast/taint-tracking/filter') +const { getToolCallResultContent } = require('../../../../src/llmobs/plugins/ai/util') const { NODE_MAJOR } = require('../../../../../../version') @@ -19,6 +20,9 @@ const { MOCK_OBJECT, } = require('../../util') +const UNPARSABLE_TOOL_RESULT = '[Unparsable Tool Result]' +const UNSUPPORTED_TOOL_RESULT = '[Unsupported Tool Result]' + // ai<4.0.2 is not supported in CommonJS with Node.js < 22 const range = NODE_MAJOR < 22 ? '>=4.0.2 <7.0.0' : '>=4.0.0 <7.0.0' @@ -63,12 +67,17 @@ function getAiSdkAnthropicOrGoogleRange (vercelAiVersion) { /** * @param {string} versionRange - * @param {(version: string, realVersion: string, openaiVersion: string) => void} callback + * @param {( + * version: string, + * realVersion: string, + * openaiVersionKey: string, + * openaiVersion: string + * ) => void} callback */ function withAiSdkOpenAiVersions (versionRange, callback) { withVersions('ai', 'ai', versionRange, (version, _, realVersion) => { - withVersions('ai', '@ai-sdk/openai', getAiSdkOpenAiRange(realVersion), openaiVersion => { - callback(version, realVersion, openaiVersion) + withVersions('ai', '@ai-sdk/openai', getAiSdkOpenAiRange(realVersion), (openaiVersionKey, _, openaiVersion) => { + callback(version, realVersion, openaiVersionKey, openaiVersion) }) }) } @@ -79,6 +88,109 @@ const MOCK_TELEMETRY_METADATA = { conversationId: 'convAbc123', } +const TOOL_RESULT_SCENARIOS = [ + { + name: 'multipart content', + execute: () => 'result', + createModelOutput: () => ({ + type: 'content', + value: [ + { type: 'text', text: 'before' }, + { type: 'media', mediaType: 'image/png', data: 'private-image-data' }, + { type: 'media', mediaType: 'application/pdf', data: 'private-file-data' }, + { type: 'file-data', mediaType: 'application/pdf', data: 'private-file-data' }, + { type: 'file-id', fileId: 'private-file-id' }, + { type: 'image-data', mediaType: 'image/png', data: 'private-image-data' }, + { type: 'image-file-id', fileId: 'private-image-id' }, + { type: 'custom', providerOptions: { secret: 'provider-data' } }, + { type: 'text', text: 'after' }, + ], + }), + expectedContent: 'before[Image][File][File][File][Image][Image][Custom Content]after', + }, + { + name: 'text', + execute: () => 'result', + expectedContent: 'result', + }, + { + name: 'JSON', + execute: () => ({ answer: 42 }), + expectedContent: '{"answer":42}', + }, + { + name: 'error text', + execute: () => { + throw new Error('tool failure') + }, + expectedContent: 'tool failure', + }, + { + name: 'error JSON', + execute: () => 'result', + createModelOutput: () => ({ + type: 'error-json', + value: { message: 'tool failure' }, + }), + expectedContent: '{"message":"tool failure"}', + }, + { + name: 'empty multipart content', + execute: () => 'result', + createModelOutput: () => ({ type: 'content', value: [] }), + expectedContent: '', + }, + { + name: 'denied execution', + openaiRange: '>=3.0.0', + execute: () => 'result', + createModelOutput: () => ({ + type: 'execution-denied', + reason: 'Approval rejected', + }), + expectedContent: 'Approval rejected', + }, + { + name: 'denied execution without a reason', + openaiRange: '>=3.0.0', + execute: () => 'result', + createModelOutput: () => ({ type: 'execution-denied' }), + expectedContent: '[Tool Execution Denied]', + }, + { + name: 'malformed multipart content', + execute: () => 'result', + createModelOutput: () => ({ + type: 'content', + value: [{ type: 'unknown' }], + }), + expectedContent: UNPARSABLE_TOOL_RESULT, + }, +] + +const LEGACY_TOOL_RESULT_SCENARIOS = [ + { + name: 'empty legacy result', + execute: () => '', + expectedContent: '', + }, + { + name: 'zero legacy result', + execute: () => 0, + expectedContent: '0', + }, + { + name: 'false legacy result', + execute: () => false, + expectedContent: 'false', + }, + { + name: 'null legacy result', + execute: () => null, + expectedContent: 'null', + }, +] + describe('Plugin', () => { useEnv({ OPENAI_API_KEY: '', @@ -1159,6 +1271,192 @@ describe('Plugin', () => { }) }) + describe('tool result formatting', () => { + withAiSdkOpenAiVersions(range, (version, realVersion, openaiVersionKey, openaiVersion) => { + let ai + let openai + + beforeEach(() => { + ai = require(`../../../../../../versions/ai@${version}`).get() + + const OpenAIModule = require(`../../../../../../versions/@ai-sdk/openai@${openaiVersionKey}`) + const OpenAI = OpenAIModule.get() + openai = OpenAI.createOpenAI({ + baseURL: 'http://127.0.0.1:9126/vcr/openai', + compatibility: 'strict', + }) + }) + + /** + * @param {{ + * execute: () => unknown, + * createModelOutput?: () => unknown, + * expectedContent: string, + * openaiRange?: string + * }} scenario + */ + async function assertToolResult (scenario) { + const isLegacy = semifies(realVersion, '<5.0.0') + const schema = ai.jsonSchema({ + type: 'object', + properties: {}, + }) + let tool + if (isLegacy) { + tool = ai.tool({ + id: 'testTool', + description: 'Run the test tool and return its result', + parameters: schema, + execute: scenario.execute, + }) + } else { + tool = ai.tool({ + description: 'Run the test tool and return its result', + inputSchema: schema, + execute: scenario.execute, + toModelOutput: scenario.createModelOutput, + }) + } + + const options = { + // Chat Completions serializes every tool result as text. The Responses API rejects some valid AI SDK + // result variants before the instrumented request can capture their formatted value. + model: openai.chat('gpt-4o-mini'), + prompt: 'Run the test tool', + tools: isLegacy ? [tool] : { testTool: tool }, + } + if (isLegacy) { + options.maxSteps = 2 + } else { + options.stopWhen = ai.stepCountIs(2) + } + if (semifies(openaiVersion, '>=2.0.50')) { + options.providerOptions = { + openai: { + store: false, + }, + } + } + + const result = await ai.generateText(options) + const toolCallId = result.steps[0].toolCalls[0].toolCallId + + const { apmSpans, llmobsSpans } = await getEvents(4) + let finalModelSpan + let finalModelApmSpan + let workflowSpan + for (const span of llmobsSpans) { + if (span.name === 'doGenerate') { + finalModelSpan = span + } else if (span.name === 'generateText') { + workflowSpan = span + } + } + for (const span of apmSpans) { + if (span.name === 'ai.generateText.doGenerate') finalModelApmSpan = span + } + + assertLlmObsSpanEvent(finalModelSpan, { + span: finalModelApmSpan, + parentId: workflowSpan.span_id, + spanKind: 'llm', + modelName: 'gpt-4o-mini', + modelProvider: 'openai', + name: 'doGenerate', + inputMessages: [ + { content: 'Run the test tool', role: 'user' }, + { + content: '', + role: 'assistant', + tool_calls: [{ + tool_id: toolCallId, + name: 'testTool', + arguments: {}, + type: 'function', + }], + }, + { + content: scenario.expectedContent, + role: 'tool', + tool_id: toolCallId, + }, + ], + outputMessages: [MOCK_OBJECT], + metrics: { input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, total_tokens: MOCK_NUMBER }, + tags: { ml_app: 'test', integration: 'ai' }, + }) + } + + const scenarios = semifies(realVersion, '<5.0.0') + ? LEGACY_TOOL_RESULT_SCENARIOS + : TOOL_RESULT_SCENARIOS.filter(({ openaiRange }) => !openaiRange || semifies(openaiVersion, openaiRange)) + for (const scenario of scenarios) { + it(`formats ${scenario.name} produced by the AI SDK`, async () => { + await assertToolResult(scenario) + }) + } + }) + + it('falls back for values rejected before model invocation', () => { + const circular = {} + circular.self = circular + + const malformedOutputs = [ + null, + 'text', + { type: 'text' }, + { type: 'text', value: 42 }, + { type: 'json', value: undefined }, + { type: 'json', value: Symbol('result') }, + { type: 'json', value: 1n }, + { type: 'json', value: circular }, + { type: 'content', value: {} }, + { type: 'content', value: [null] }, + { type: 'content', value: [{ type: 'text' }] }, + { type: 'content', value: [{ type: 'media' }] }, + { type: 'execution-denied', reason: {} }, + { type: 'unknown', value: 'result' }, + ] + + for (const output of malformedOutputs) { + assert.strictEqual( + getToolCallResultContent({ output }), + UNPARSABLE_TOOL_RESULT + ) + } + + assert.strictEqual(getToolCallResultContent(), UNPARSABLE_TOOL_RESULT) + assert.strictEqual(getToolCallResultContent(null), UNPARSABLE_TOOL_RESULT) + assert.strictEqual(getToolCallResultContent('result'), UNPARSABLE_TOOL_RESULT) + assert.strictEqual(getToolCallResultContent({}), UNSUPPORTED_TOOL_RESULT) + }) + + it('does not throw for external objects rejected before model invocation', () => { + const throwingContent = new Proxy({}, { + get () { + throw new Error('unexpected content access') + }, + }) + const throwingOutput = new Proxy({}, { + get () { + throw new Error('unexpected output access') + }, + }) + const throwingParts = new Proxy([], { + get () { + throw new Error('unexpected content part access') + }, + }) + + assert.strictEqual(getToolCallResultContent(throwingContent), UNPARSABLE_TOOL_RESULT) + assert.strictEqual(getToolCallResultContent({ output: throwingOutput }), UNPARSABLE_TOOL_RESULT) + assert.strictEqual( + getToolCallResultContent({ output: { type: 'content', value: throwingParts } }), + UNPARSABLE_TOOL_RESULT + ) + }) + }) + describe('prompt cache token capture', () => { // Both @ai-sdk/amazon-bedrock and @ai-sdk/anthropic use globalThis.fetch // (not node:http), so nock cannot intercept them. Each provider's diff --git a/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js index 52a686a5c2..dd3b400ca2 100644 --- a/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js @@ -11,6 +11,7 @@ const { MOCK_STRING, useLlmObs, MOCK_NUMBER, + MOCK_OBJECT, } = require('../../util') /** @@ -946,6 +947,137 @@ describe('Plugin', () => { }) }) + describe('tool result formatting', () => { + withAiSdkOpenAiVersions((version, _, openaiVersion) => { + let ai + let openai + + beforeEach(() => { + ai = require(`../../../../../../versions/ai@${version}`).get() + + const OpenAI = require(`../../../../../../versions/@ai-sdk/openai@${openaiVersion}`).get() + openai = OpenAI.createOpenAI({ + baseURL: 'http://127.0.0.1:9126/vcr/openai', + compatibility: 'strict', + }) + }) + + it('formats file result parts produced by the AI SDK', async () => { + const result = await ai.generateText({ + // Chat Completions serializes every tool result as text. The Responses API validates file contents before + // the instrumented request can capture their formatted value. + model: openai.chat('gpt-4o-mini'), + prompt: 'Run the test tool', + tools: { + testTool: ai.tool({ + description: 'Run the test tool and return its result', + inputSchema: ai.jsonSchema({ + type: 'object', + properties: {}, + }), + execute: () => 'result', + toModelOutput: () => ({ + type: 'content', + value: [ + { type: 'text', text: 'before' }, + { + type: 'file', + mediaType: 'image/png', + data: { type: 'data', data: 'cHJpdmF0ZS1pbWFnZS1kYXRh' }, + }, + { + type: 'file', + mediaType: 'application/pdf', + data: { type: 'data', data: 'cHJpdmF0ZS1maWxlLWRhdGE=' }, + }, + { + type: 'file', + mediaType: 'application/pdf', + data: { type: 'reference', reference: { test: 'private-file-id' } }, + }, + { + type: 'file', + mediaType: 'image/png', + data: { type: 'reference', reference: { test: 'private-image-id' } }, + }, + { type: 'custom', providerOptions: { secret: 'provider-data' } }, + { type: 'text', text: 'after' }, + ], + }), + }), + }, + stopWhen: ai.stepCountIs(2), + providerOptions: { + openai: { + store: false, + }, + }, + }) + const toolCallId = result.steps[0].toolCalls[0].toolCallId + + const { apmSpans, llmobsSpans } = await getEvents(6) + let finalModelSpan + let finalModelApmSpan + let finalStepSpan + for (const span of llmobsSpans) { + if (span.name === 'languageModelCall') { + finalModelSpan = span + } else if (span.name === 'step') { + finalStepSpan = span + } + } + for (const span of apmSpans) { + if (span.name === 'languageModelCall') finalModelApmSpan = span + } + + assertLlmObsSpanEvent(finalModelSpan, { + span: finalModelApmSpan, + parentId: finalStepSpan.span_id, + spanKind: 'llm', + modelName: 'gpt-4o-mini', + modelProvider: 'openai', + name: 'languageModelCall', + inputMessages: [ + { content: 'Run the test tool', role: 'user' }, + { + role: 'assistant', + tool_calls: [{ + tool_id: toolCallId, + name: 'testTool', + arguments: {}, + type: 'function', + }], + }, + { + content: 'before[Image][File][File][Image][Custom Content]after', + role: 'tool', + tool_id: toolCallId, + }, + ], + outputMessages: [MOCK_OBJECT], + toolDefinitions: [{ + name: 'testTool', + description: 'Run the test tool and return its result', + schema: { + type: 'object', + properties: {}, + required: [], + }, + }], + metadata: {}, + metrics: { + input_tokens: MOCK_NUMBER, + cache_write_input_tokens: 0, + cache_read_input_tokens: 0, + output_tokens: MOCK_NUMBER, + reasoning_output_tokens: 0, + }, + tags: { ml_app: 'test', integration: 'ai' }, + }) + }) + }) + }) + describe('prompt cache token capture', () => { function makeMockFetch (scenario) { const fixture = require(`../../../../../datadog-plugin-ai/test/resources/${scenario}.json`)