feat: Instrument Anthropic SDK tool runner (#1833)

Fixes
#1827

Author: lforst

.changeset/slimy-hands-run.md

@@ -0,0 +1,5 @@
+---
+"braintrust": minor
+---
+
+feat: Instrument Anthropic SDK tool runner

e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.log-payloads.json

@@ -429,5 +429,167 @@
     "name": "anthropic.messages.create",
     "output": "1 - one\n2 - two\n3 - three",
     "type": "llm"
+  },
+  {
+    "metadata": {
+      "operation": "beta-tool-runner"
+    },
+    "metrics": {
+      "end": 0,
+      "start": 0
+    },
+    "name": "anthropic-beta-tool-runner-operation",
+    "type": null
+  },
+  {
+    "input": [
+      {
+        "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.",
+        "role": "user"
+      }
+    ],
+    "metadata": {
+      "anthropic_tool_runner_iterations": 2,
+      "model": "claude-3-haiku-20240307",
+      "operation": "toolRunner",
+      "provider": "anthropic",
+      "stop_reason": "end_turn",
+      "stop_sequence": null
+    },
+    "metrics": {
+      "completion_tokens": 58,
+      "end": 0,
+      "prompt_cache_creation_tokens": 0,
+      "prompt_cached_tokens": 0,
+      "prompt_tokens": 829,
+      "start": 0,
+      "time_to_first_token": 0,
+      "tokens": 887
+    },
+    "name": "anthropic.beta.messages.toolRunner",
+    "output": {
+      "content": [],
+      "role": "assistant"
+    },
+    "type": "task"
+  },
+  {
+    "input": {
+      "location": "Paris, France"
+    },
+    "metadata": {
+      "provider": "anthropic"
+    },
+    "metrics": {
+      "end": 0,
+      "start": 0
+    },
+    "name": "tool: get_weather",
+    "output": "The weather in Paris, France is 18C and sunny.",
+    "type": "tool"
+  },
+  {
+    "metadata": null,
+    "metrics": {
+      "end": 0,
+      "start": 0
+    },
+    "name": "get_weather.lookup",
+    "type": null
+  },
+  {
+    "input": [
+      {
+        "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.",
+        "role": "user"
+      }
+    ],
+    "metadata": {
+      "model": "claude-3-haiku-20240307",
+      "provider": "anthropic",
+      "stop_reason": "tool_use",
+      "stop_sequence": null
+    },
+    "metrics": {
+      "completion_tokens": 55,
+      "end": 0,
+      "prompt_cache_creation_tokens": 0,
+      "prompt_cached_tokens": 0,
+      "prompt_tokens": 375,
+      "start": 0,
+      "time_to_first_token": 0,
+      "tokens": 430
+    },
+    "name": "anthropic.messages.create",
+    "output": {
+      "content": [
+        {
+          "id": "<span:3>",
+          "input": {
+            "location": "Paris, France"
+          },
+          "name": "get_weather",
+          "type": "tool_use"
+        }
+      ],
+      "role": "assistant"
+    },
+    "type": "llm"
+  },
+  {
+    "input": [
+      {
+        "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.",
+        "role": "user"
+      },
+      {
+        "content": [
+          {
+            "caller": {
+              "type": "direct"
+            },
+            "id": "<span:3>",
+            "input": {
+              "location": "Paris, France"
+            },
+            "name": "get_weather",
+            "type": "tool_use"
+          }
+        ],
+        "role": "assistant"
+      },
+      {
+        "content": [
+          {
+            "content": "The weather in Paris, France is 18C and sunny.",
+            "tool_use_id": "<tool-use-id>",
+            "type": "tool_result"
+          }
+        ],
+        "role": "user"
+      }
+    ],
+    "metadata": {
+      "model": "claude-3-haiku-20240307",
+      "provider": "anthropic",
+      "stop_reason": "end_turn",
+      "stop_sequence": null
+    },
+    "metrics": {
+      "completion_tokens": 3,
+      "end": 0,
+      "prompt_cache_creation_tokens": 0,
+      "prompt_cached_tokens": 0,
+      "prompt_tokens": 454,
+      "start": 0,
+      "time_to_first_token": 0,
+      "tokens": 457
+    },
+    "name": "anthropic.messages.create",
+    "output": {
+      "content": [],
+      "role": "assistant"
+    },
+    "type": "llm"
   }
 ]

e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-events.json

@@ -353,5 +353,119 @@
       "<span:18>"
     ],
     "type": "llm"
+  },
+  {
+    "has_input": false,
+    "has_output": false,
+    "metadata": {
+      "operation": "beta-tool-runner"
+    },
+    "metric_keys": [],
+    "name": "anthropic-beta-tool-runner-operation",
+    "root_span_id": "<span:1>",
+    "span_id": "<span:20>",
+    "span_parents": [
+      "<span:1>"
+    ],
+    "type": null
+  },
+  {
+    "has_input": true,
+    "has_output": true,
+    "metadata": {
+      "anthropic_tool_runner_iterations": 2,
+      "model": "claude-3-haiku-20240307",
+      "operation": "toolRunner",
+      "provider": "anthropic"
+    },
+    "metric_keys": [
+      "completion_tokens",
+      "prompt_cache_creation_tokens",
+      "prompt_cached_tokens",
+      "prompt_tokens",
+      "time_to_first_token",
+      "tokens"
+    ],
+    "name": "anthropic.beta.messages.toolRunner",
+    "root_span_id": "<span:1>",
+    "span_id": "<span:21>",
+    "span_parents": [
+      "<span:20>"
+    ],
+    "type": "task"
+  },
+  {
+    "has_input": true,
+    "has_output": true,
+    "metadata": {
+      "provider": "anthropic"
+    },
+    "metric_keys": [],
+    "name": "tool: get_weather",
+    "root_span_id": "<span:1>",
+    "span_id": "<span:22>",
+    "span_parents": [
+      "<span:21>"
+    ],
+    "type": "tool"
+  },
+  {
+    "has_input": false,
+    "has_output": false,
+    "metadata": null,
+    "metric_keys": [],
+    "name": "get_weather.lookup",
+    "root_span_id": "<span:1>",
+    "span_id": "<span:23>",
+    "span_parents": [
+      "<span:22>"
+    ],
+    "type": null
+  },
+  {
+    "has_input": true,
+    "has_output": true,
+    "metadata": {
+      "model": "claude-3-haiku-20240307",
+      "provider": "anthropic"
+    },
+    "metric_keys": [
+      "completion_tokens",
+      "prompt_cache_creation_tokens",
+      "prompt_cached_tokens",
+      "prompt_tokens",
+      "time_to_first_token",
+      "tokens"
+    ],
+    "name": "anthropic.messages.create",
+    "root_span_id": "<span:1>",
+    "span_id": "<span:24>",
+    "span_parents": [
+      "<span:21>"
+    ],
+    "type": "llm"
+  },
+  {
+    "has_input": true,
+    "has_output": true,
+    "metadata": {
+      "model": "claude-3-haiku-20240307",
+      "provider": "anthropic"
+    },
+    "metric_keys": [
+      "completion_tokens",
+      "prompt_cache_creation_tokens",
+      "prompt_cached_tokens",
+      "prompt_tokens",
+      "time_to_first_token",
+      "tokens"
+    ],
+    "name": "anthropic.messages.create",
+    "root_span_id": "<span:1>",
+    "span_id": "<span:25>",
+    "span_parents": [
+      "<span:21>"
+    ],
+    "type": "llm"
   }
 ]