diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/CHANGELOG.md b/instrumentation-genai/opentelemetry-instrumentation-anthropic/CHANGELOG.md index ba164c0ebc..57f179e00c 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-anthropic/CHANGELOG.md +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Add async support for Anthropic instrumentation with `AsyncMessages.create` and `AsyncMessages.stream` + ([#4156](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4156)) - Initial implementation of Anthropic instrumentation ([#3978](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3978)) - Implement sync `Messages.create` instrumentation with GenAI semantic convention attributes diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/__init__.py b/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/__init__.py index bf76798462..6ae5b59287 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/__init__.py +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/__init__.py @@ -54,7 +54,11 @@ ) from opentelemetry.instrumentation.anthropic.package import _instruments -from opentelemetry.instrumentation.anthropic.patch import messages_create +from opentelemetry.instrumentation.anthropic.patch import ( + async_messages_create, + async_messages_stream, + messages_create, +) from opentelemetry.instrumentation.instrumentor import BaseInstrumentor from opentelemetry.instrumentation.utils import unwrap from opentelemetry.util.genai.handler import TelemetryHandler @@ -103,6 +107,20 @@ def _instrument(self, **kwargs: Any) -> None: wrapper=messages_create(handler), ) + # Patch AsyncMessages.create + wrap_function_wrapper( + module="anthropic.resources.messages", + name="AsyncMessages.create", + wrapper=async_messages_create(handler), + ) + + # Patch AsyncMessages.stream + wrap_function_wrapper( + module="anthropic.resources.messages", + name="AsyncMessages.stream", + wrapper=async_messages_stream(handler), + ) + def _uninstrument(self, **kwargs: Any) -> None: """Disable Anthropic instrumentation. @@ -114,3 +132,11 @@ def _uninstrument(self, **kwargs: Any) -> None: anthropic.resources.messages.Messages, # pyright: ignore[reportAttributeAccessIssue,reportUnknownMemberType,reportUnknownArgumentType] "create", ) + unwrap( + anthropic.resources.messages.AsyncMessages, # pyright: ignore[reportAttributeAccessIssue,reportUnknownMemberType,reportUnknownArgumentType] + "create", + ) + unwrap( + anthropic.resources.messages.AsyncMessages, # pyright: ignore[reportAttributeAccessIssue,reportUnknownMemberType,reportUnknownArgumentType] + "stream", + ) diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/patch.py b/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/patch.py index 0562d638e2..945272d925 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/patch.py +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/patch.py @@ -14,22 +14,30 @@ """Patching functions for Anthropic instrumentation.""" -from typing import TYPE_CHECKING, Any, Callable +from typing import TYPE_CHECKING, Any, Callable, Coroutine, Union from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, ) from opentelemetry.util.genai.handler import TelemetryHandler -from opentelemetry.util.genai.types import LLMInvocation +from opentelemetry.util.genai.types import Error, LLMInvocation from .utils import ( + AsyncMessageStreamManagerWrapper, + AsyncStreamWrapper, + MessageWrapper, extract_params, get_llm_request_attributes, ) if TYPE_CHECKING: - from anthropic.resources.messages import Messages - from anthropic.types import Message + from anthropic._streaming import AsyncStream + from anthropic.lib.streaming import AsyncMessageStreamManager + from anthropic.resources.messages import AsyncMessages, Messages + from anthropic.types import Message, RawMessageStreamEvent + + +ANTHROPIC = "anthropic" def messages_create( @@ -45,15 +53,18 @@ def traced_method( ) -> "Message": params = extract_params(*args, **kwargs) attributes = get_llm_request_attributes(params, instance) - request_model = str( - attributes.get(GenAIAttributes.GEN_AI_REQUEST_MODEL) - or params.model - or "unknown" + request_model_attribute = attributes.get( + GenAIAttributes.GEN_AI_REQUEST_MODEL + ) + request_model = ( + request_model_attribute + if isinstance(request_model_attribute, str) + else params.model or "" ) invocation = LLMInvocation( request_model=request_model, - provider="anthropic", + provider=ANTHROPIC, attributes=attributes, ) @@ -76,3 +87,107 @@ def traced_method( return result return traced_method + + +def async_messages_stream( + handler: TelemetryHandler, +) -> Callable[..., "AsyncMessageStreamManager"]: + """Wrap the `stream` method of the `AsyncMessages` class to trace it.""" + + def traced_method( + wrapped: Callable[..., "AsyncMessageStreamManager"], + instance: "AsyncMessages", + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> AsyncMessageStreamManagerWrapper: + params = extract_params(*args, **kwargs) + attributes = get_llm_request_attributes(params, instance) # type: ignore[arg-type] + request_model_attribute = attributes.get( + GenAIAttributes.GEN_AI_REQUEST_MODEL + ) + request_model = ( + request_model_attribute + if isinstance(request_model_attribute, str) + else params.model or "" + ) + + invocation = LLMInvocation( + request_model=request_model, + provider=ANTHROPIC, + attributes=attributes, + ) + + # Start the span before calling the wrapped method + handler.start_llm(invocation) + try: + result = wrapped(*args, **kwargs) + # Return wrapped AsyncMessageStreamManager + return AsyncMessageStreamManagerWrapper( + result, handler, invocation + ) + except Exception as exc: + handler.fail_llm( + invocation, Error(message=str(exc), type=type(exc)) + ) + raise + + return traced_method # type: ignore[return-value] + + +def async_messages_create( + handler: TelemetryHandler, +) -> Callable[ + ..., + Coroutine[ + Any, Any, Union["Message", "AsyncStream[RawMessageStreamEvent]"] + ], +]: + """Wrap the `create` method of the `AsyncMessages` class to trace it.""" + + async def traced_method( + wrapped: Callable[ + ..., + Coroutine[ + Any, + Any, + Union["Message", "AsyncStream[RawMessageStreamEvent]"], + ], + ], + instance: "AsyncMessages", + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Union["Message", AsyncStreamWrapper]: + params = extract_params(*args, **kwargs) + attributes = get_llm_request_attributes(params, instance) # type: ignore[arg-type] + request_model_attribute = attributes.get( + GenAIAttributes.GEN_AI_REQUEST_MODEL + ) + request_model = ( + request_model_attribute + if isinstance(request_model_attribute, str) + else params.model or "" + ) + + invocation = LLMInvocation( + request_model=request_model, + provider=ANTHROPIC, + attributes=attributes, + ) + + is_streaming = kwargs.get("stream", False) + + # Use manual lifecycle management for both streaming and non-streaming + handler.start_llm(invocation) + try: + result = await wrapped(*args, **kwargs) + if is_streaming: + return AsyncStreamWrapper(result, handler, invocation) # type: ignore[arg-type] + wrapper = MessageWrapper(result, handler, invocation) # type: ignore[arg-type] + return wrapper.message + except Exception as exc: + handler.fail_llm( + invocation, Error(message=str(exc), type=type(exc)) + ) + raise + + return traced_method # type: ignore[return-value] diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/utils.py b/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/utils.py index 4c2003f441..4a21b05d7e 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/utils.py +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/src/opentelemetry/instrumentation/anthropic/utils.py @@ -17,7 +17,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Optional, Sequence +from typing import TYPE_CHECKING, Any, AsyncIterator, Optional, Sequence from urllib.parse import urlparse from opentelemetry.semconv._incubating.attributes import ( @@ -26,26 +26,104 @@ from opentelemetry.semconv._incubating.attributes import ( server_attributes as ServerAttributes, ) +from opentelemetry.util.genai.handler import TelemetryHandler +from opentelemetry.util.genai.types import Error, LLMInvocation from opentelemetry.util.types import AttributeValue if TYPE_CHECKING: + from anthropic._streaming import AsyncStream + from anthropic.lib.streaming import ( + AsyncMessageStream, + AsyncMessageStreamManager, + ) from anthropic.resources.messages import Messages + from anthropic.types import Message, RawMessageStreamEvent @dataclass -class MessageCreateParams: - """Parameters extracted from Messages.create() call.""" +class MessageRequestParams: + """Parameters extracted from Anthropic Messages API calls.""" model: str | None = None max_tokens: int | None = None temperature: float | None = None - top_p: float | None = None top_k: int | None = None + top_p: float | None = None stop_sequences: Sequence[str] | None = None +_GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS = ( + "gen_ai.usage.cache_creation.input_tokens" +) +_GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS = "gen_ai.usage.cache_read.input_tokens" + + +def _normalize_finish_reason(stop_reason: str | None) -> str | None: + """Map Anthropic stop reasons to GenAI semantic convention values.""" + if stop_reason is None: + return None + + normalized = { + "end_turn": "stop", + "stop_sequence": "stop", + "max_tokens": "length", + "tool_use": "tool_calls", + }.get(stop_reason) + return normalized or stop_reason + + +def _as_int(value: Any) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + return None + + +def _extract_usage_tokens( + usage: Any | None, +) -> tuple[int | None, int | None, int | None, int | None]: + """Extract Anthropic usage fields and compute semconv input tokens. + + Returns `(total_input_tokens, output_tokens, cache_creation_input_tokens, cache_read_input_tokens)`. + """ + if usage is None: + return None, None, None, None + + input_tokens = _as_int(getattr(usage, "input_tokens", None)) + cache_creation_input_tokens = _as_int( + getattr(usage, "cache_creation_input_tokens", None) + ) + cache_read_input_tokens = _as_int( + getattr(usage, "cache_read_input_tokens", None) + ) + output_tokens = _as_int(getattr(usage, "output_tokens", None)) + + if ( + input_tokens is None + and cache_creation_input_tokens is None + and cache_read_input_tokens is None + ): + total_input_tokens = None + else: + total_input_tokens = ( + (input_tokens or 0) + + (cache_creation_input_tokens or 0) + + (cache_read_input_tokens or 0) + ) + + return ( + total_input_tokens, + output_tokens, + cache_creation_input_tokens, + cache_read_input_tokens, + ) + + # Use parameter signature from -# https://github.com/anthropics/anthropic-sdk-python/blob/9b5ab24ba17bcd5e762e5a5fd69bb3c17b100aaa/src/anthropic/resources/messages/messages.py#L92 +# https://github.com/anthropics/anthropic-sdk-python/blob/9b5ab24ba17bcd5e762e5a5fd69bb3c17b100aaa/src/anthropic/resources/messages/messages.py#L896 +# https://github.com/anthropics/anthropic-sdk-python/blob/9b5ab24ba17bcd5e762e5a5fd69bb3c17b100aaa/src/anthropic/resources/messages/messages.py#L2080 +# https://github.com/anthropics/anthropic-sdk-python/blob/9b5ab24ba17bcd5e762e5a5fd69bb3c17b100aaa/src/anthropic/resources/messages/messages.py#L2147 # to handle named vs positional args robustly def extract_params( # pylint: disable=too-many-locals *, @@ -61,16 +139,16 @@ def extract_params( # pylint: disable=too-many-locals thinking: Any | None = None, tool_choice: Any | None = None, tools: Any | None = None, - top_p: float | None = None, top_k: int | None = None, + top_p: float | None = None, extra_headers: Any | None = None, extra_query: Any | None = None, extra_body: Any | None = None, timeout: Any | None = None, **_kwargs: Any, -) -> MessageCreateParams: - """Extract relevant parameters from Messages.create() arguments.""" - return MessageCreateParams( +) -> MessageRequestParams: + """Extract relevant parameters from Anthropic Messages API arguments.""" + return MessageRequestParams( model=model, max_tokens=max_tokens, temperature=temperature, @@ -104,9 +182,9 @@ def set_server_address_and_port( def get_llm_request_attributes( - params: MessageCreateParams, client_instance: "Messages" + params: MessageRequestParams, client_instance: "Messages" ) -> dict[str, AttributeValue]: - """Extract LLM request attributes from MessageCreateParams. + """Extract LLM request attributes from MessageRequestParams. Returns a dictionary of OpenTelemetry semantic convention attributes for LLM requests. The attributes follow the GenAI semantic conventions (gen_ai.*) and server semantic @@ -143,3 +221,278 @@ def get_llm_request_attributes( # Filter out None values return {k: v for k, v in attributes.items() if v is not None} + + +class MessageWrapper: + """Wrapper for non-streaming Message response that handles telemetry. + + This wrapper extracts telemetry data from the response and finalizes + the span immediately since the response is complete. + """ + + def __init__( + self, + message: "Message", + handler: TelemetryHandler, + invocation: LLMInvocation, + ): + self._message = message + self._extract_and_finalize(handler, invocation) + + def _extract_and_finalize( + self, handler: TelemetryHandler, invocation: LLMInvocation + ) -> None: + """Extract response data and finalize the span.""" + if self._message.model: + invocation.response_model_name = self._message.model + + if self._message.id: + invocation.response_id = self._message.id + + finish_reason = _normalize_finish_reason(self._message.stop_reason) + if finish_reason: + invocation.finish_reasons = [finish_reason] + + if self._message.usage: + ( + input_tokens, + output_tokens, + cache_creation_input_tokens, + cache_read_input_tokens, + ) = _extract_usage_tokens(self._message.usage) + invocation.input_tokens = input_tokens + invocation.output_tokens = output_tokens + if cache_creation_input_tokens is not None: + invocation.attributes[ + _GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS + ] = cache_creation_input_tokens + if cache_read_input_tokens is not None: + invocation.attributes[ + _GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS + ] = cache_read_input_tokens + + handler.stop_llm(invocation) + + @property + def message(self) -> "Message": + """Return the wrapped Message object.""" + return self._message + + +class AsyncStreamWrapper(AsyncIterator["RawMessageStreamEvent"]): + """Wrapper for async Anthropic Stream that handles telemetry. + + This wrapper wraps AsyncStream[RawMessageStreamEvent] returned by + AsyncMessages.create(stream=True). It processes streaming chunks, + extracts telemetry data, and ensures the span is properly ended + when the stream is consumed. + """ + + def __init__( + self, + stream: "AsyncStream[RawMessageStreamEvent]", + handler: TelemetryHandler, + invocation: LLMInvocation, + ): + self._stream = stream + self._handler = handler + self._invocation = invocation + self._response_id: Optional[str] = None + self._response_model: Optional[str] = None + self._stop_reason: Optional[str] = None + self._input_tokens: Optional[int] = None + self._output_tokens: Optional[int] = None + self._cache_creation_input_tokens: Optional[int] = None + self._cache_read_input_tokens: Optional[int] = None + self._finalized = False + + def _update_usage(self, usage: Any | None) -> None: + ( + input_tokens, + output_tokens, + cache_creation_input_tokens, + cache_read_input_tokens, + ) = _extract_usage_tokens(usage) + if input_tokens is not None: + self._input_tokens = input_tokens + if output_tokens is not None: + self._output_tokens = output_tokens + if cache_creation_input_tokens is not None: + self._cache_creation_input_tokens = cache_creation_input_tokens + if cache_read_input_tokens is not None: + self._cache_read_input_tokens = cache_read_input_tokens + + def _process_chunk(self, chunk: "RawMessageStreamEvent") -> None: + """Extract telemetry data from a streaming chunk.""" + # Handle message_start event - contains initial message info + if chunk.type == "message_start": + message = getattr(chunk, "message", None) + if message: + if hasattr(message, "id") and message.id: + self._response_id = message.id + if hasattr(message, "model") and message.model: + self._response_model = message.model + # message_start also contains initial usage with input_tokens + if hasattr(message, "usage") and message.usage: + self._update_usage(message.usage) + + # Handle message_delta event - contains stop_reason and output token usage + elif chunk.type == "message_delta": + delta = getattr(chunk, "delta", None) + if delta and hasattr(delta, "stop_reason") and delta.stop_reason: + self._stop_reason = _normalize_finish_reason(delta.stop_reason) + # message_delta contains usage with output tokens and may repeat input/cache tokens + usage = getattr(chunk, "usage", None) + self._update_usage(usage) + + def _finalize_invocation(self) -> None: + """Update invocation with collected data and stop the span.""" + if self._finalized: + return + + if self._response_model: + self._invocation.response_model_name = self._response_model + if self._response_id: + self._invocation.response_id = self._response_id + if self._stop_reason: + self._invocation.finish_reasons = [self._stop_reason] + if self._input_tokens is not None: + self._invocation.input_tokens = self._input_tokens + if self._output_tokens is not None: + self._invocation.output_tokens = self._output_tokens + if self._cache_creation_input_tokens is not None: + self._invocation.attributes[ + _GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS + ] = self._cache_creation_input_tokens + if self._cache_read_input_tokens is not None: + self._invocation.attributes[ + _GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS + ] = self._cache_read_input_tokens + + self._handler.stop_llm(self._invocation) + self._finalized = True + + def __aiter__(self) -> "AsyncStreamWrapper": + return self + + async def __anext__(self) -> "RawMessageStreamEvent": + try: + chunk = await self._stream.__anext__() + self._process_chunk(chunk) + return chunk + except StopAsyncIteration: + self._finalize_invocation() + raise + except Exception as exc: + self._handler.fail_llm( + self._invocation, Error(message=str(exc), type=type(exc)) + ) + raise + + async def __aenter__(self) -> "AsyncStreamWrapper": + return self + + async def __aexit__( + self, exc_type: Any, exc_val: Any, exc_tb: Any + ) -> bool: + await self.close() + return False + + async def close(self) -> None: + """Close the underlying stream and finalize telemetry.""" + if hasattr(self._stream, "close"): + await self._stream.close() + self._finalize_invocation() + + +class AsyncMessageStreamManagerWrapper: + """Wrapper for AsyncMessageStreamManager that handles telemetry. + + This wrapper wraps the AsyncMessageStreamManager async context manager returned by + AsyncMessages.stream(). It extracts telemetry data from the final message + when the context exits. + """ + + def __init__( + self, + stream_manager: "AsyncMessageStreamManager", + handler: TelemetryHandler, + invocation: LLMInvocation, + ): + self._stream_manager = stream_manager + self._handler = handler + self._invocation = invocation + self._message_stream: Optional["AsyncMessageStream"] = None + + async def __aenter__(self) -> "AsyncMessageStream": + """Enter the async context and return the underlying AsyncMessageStream.""" + try: + self._message_stream = await self._stream_manager.__aenter__() + return self._message_stream + except Exception as exc: + # Handle errors during context entry (e.g., connection errors) + self._handler.fail_llm( + self._invocation, + Error(message=str(exc), type=type(exc)), + ) + raise + + async def __aexit__( + self, exc_type: Any, exc_val: Any, exc_tb: Any + ) -> bool: + """Exit the async context, extract telemetry, and finalize the span.""" + # Extract telemetry from the final message before exiting + if self._message_stream is not None and exc_type is None: + await self._extract_telemetry_from_stream() + self._handler.stop_llm(self._invocation) + elif exc_type is not None: + # Handle error case + self._handler.fail_llm( + self._invocation, + Error( + message=str(exc_val) if exc_val else str(exc_type), + type=exc_type, + ), + ) + # Always exit the underlying stream manager + return await self._stream_manager.__aexit__(exc_type, exc_val, exc_tb) # type: ignore[return-value] + + async def _extract_telemetry_from_stream(self) -> None: + """Extract telemetry data from the AsyncMessageStream's final message.""" + if self._message_stream is None: + return + + try: + # get_final_message() returns the accumulated Message object + final_message = await self._message_stream.get_final_message() + + if final_message.model: + self._invocation.response_model_name = final_message.model + + if final_message.id: + self._invocation.response_id = final_message.id + + finish_reason = _normalize_finish_reason(final_message.stop_reason) + if finish_reason: + self._invocation.finish_reasons = [finish_reason] + + if final_message.usage: + ( + input_tokens, + output_tokens, + cache_creation_input_tokens, + cache_read_input_tokens, + ) = _extract_usage_tokens(final_message.usage) + self._invocation.input_tokens = input_tokens + self._invocation.output_tokens = output_tokens + if cache_creation_input_tokens is not None: + self._invocation.attributes[ + _GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS + ] = cache_creation_input_tokens + if cache_read_input_tokens is not None: + self._invocation.attributes[ + _GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS + ] = cache_read_input_tokens + except Exception: # pylint: disable=broad-exception-caught + # If we can't get the final message, we still want to end the span + pass diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_api_error.yaml b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_api_error.yaml new file mode 100644 index 0000000000..8b4320c700 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_api_error.yaml @@ -0,0 +1,100 @@ +interactions: +- request: + body: |- + { + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Hello" + } + ], + "model": "invalid-model-name" + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '94' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 0.75.0 + x-api-key: + - test_anthropic_api_key + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.75.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.9.6 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "type": "error", + "error": { + "type": "not_found_error", + "message": "model: invalid-model-name" + }, + "request_id": "req_011CXhfs9qeGbqNg41bnA3ia" + } + headers: + CF-RAY: + - 9c72d9541b9b6399-EWR + Connection: + - keep-alive + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - application/json + Date: + - Sun, 01 Feb 2026 16:33:10 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 455ea6be-bd92-4199-83ec-0c6b39c5c169 + cf-cache-status: + - DYNAMIC + content-length: + - '133' + request-id: + - req_011CXhfs9qeGbqNg41bnA3ia + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - '278' + x-should-retry: + - 'false' + status: + code: 404 + message: Not Found +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_basic.yaml b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_basic.yaml new file mode 100644 index 0000000000..2bd3054c76 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_basic.yaml @@ -0,0 +1,139 @@ +interactions: +- request: + body: |- + { + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Say hello in one word." + } + ], + "model": "claude-sonnet-4-20250514" + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '117' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 0.75.0 + x-api-key: + - test_anthropic_api_key + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.75.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.9.6 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "model": "claude-sonnet-4-20250514", + "id": "msg_019vpyHWetquQZLsc6zc1mhj", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "Hello!" + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 13, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": { + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 5, + "service_tier": "standard" + } + } + headers: + CF-RAY: + - 9c72d9232bb4cd7c-EWR + Connection: + - keep-alive + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - application/json + Date: + - Sun, 01 Feb 2026 16:33:03 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 455ea6be-bd92-4199-83ec-0c6b39c5c169 + anthropic-ratelimit-input-tokens-limit: + - '30000' + anthropic-ratelimit-input-tokens-remaining: + - '30000' + anthropic-ratelimit-input-tokens-reset: + - '2026-02-01T16:33:03Z' + anthropic-ratelimit-output-tokens-limit: + - '8000' + anthropic-ratelimit-output-tokens-remaining: + - '8000' + anthropic-ratelimit-output-tokens-reset: + - '2026-02-01T16:33:03Z' + anthropic-ratelimit-requests-limit: + - '50' + anthropic-ratelimit-requests-remaining: + - '49' + anthropic-ratelimit-requests-reset: + - '2026-02-01T16:33:03Z' + anthropic-ratelimit-tokens-limit: + - '38000' + anthropic-ratelimit-tokens-remaining: + - '38000' + anthropic-ratelimit-tokens-reset: + - '2026-02-01T16:33:03Z' + cf-cache-status: + - DYNAMIC + content-length: + - '409' + request-id: + - req_011CXhfraJH3FJ7Vy5FSPsUE + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - '1716' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_stop_reason.yaml b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_stop_reason.yaml new file mode 100644 index 0000000000..f994f13a28 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_stop_reason.yaml @@ -0,0 +1,139 @@ +interactions: +- request: + body: |- + { + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Say hi." + } + ], + "model": "claude-sonnet-4-20250514" + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '102' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 0.75.0 + x-api-key: + - test_anthropic_api_key + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.75.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.9.6 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "model": "claude-sonnet-4-20250514", + "id": "msg_011FM3Z7B1Mse9MMddZcrsJe", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "Hi! How are you doing today?" + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 10, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": { + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 11, + "service_tier": "standard" + } + } + headers: + CF-RAY: + - 9c72d9431acc43fb-EWR + Connection: + - keep-alive + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - application/json + Date: + - Sun, 01 Feb 2026 16:33:08 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 455ea6be-bd92-4199-83ec-0c6b39c5c169 + anthropic-ratelimit-input-tokens-limit: + - '30000' + anthropic-ratelimit-input-tokens-remaining: + - '30000' + anthropic-ratelimit-input-tokens-reset: + - '2026-02-01T16:33:08Z' + anthropic-ratelimit-output-tokens-limit: + - '8000' + anthropic-ratelimit-output-tokens-remaining: + - '8000' + anthropic-ratelimit-output-tokens-reset: + - '2026-02-01T16:33:08Z' + anthropic-ratelimit-requests-limit: + - '50' + anthropic-ratelimit-requests-remaining: + - '49' + anthropic-ratelimit-requests-reset: + - '2026-02-01T16:33:08Z' + anthropic-ratelimit-tokens-limit: + - '38000' + anthropic-ratelimit-tokens-remaining: + - '38000' + anthropic-ratelimit-tokens-reset: + - '2026-02-01T16:33:08Z' + cf-cache-status: + - DYNAMIC + content-length: + - '432' + request-id: + - req_011CXhfrx8C5B1XrBu2F8x8n + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - '1182' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_streaming.yaml b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_streaming.yaml new file mode 100644 index 0000000000..3b01807a60 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_streaming.yaml @@ -0,0 +1,136 @@ +interactions: +- request: + body: |- + { + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Say hello in one word." + } + ], + "model": "claude-sonnet-4-20250514", + "stream": true + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '131' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 0.75.0 + x-api-key: + - test_anthropic_api_key + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.75.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.9.6 + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |+ + event: message_start + data: {"type":"message_start","message":{"model":"claude-sonnet-4-20250514","id":"msg_01BndF5goeaRB8efAEVZn47Z","type":"message","role":"assistant","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":13,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":2,"service_tier":"standard"}} } + + event: content_block_start + data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} } + + event: ping + data: {"type": "ping"} + + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello!"} } + + event: content_block_stop + data: {"type":"content_block_stop","index":0 } + + event: message_delta + data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":13,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":5}} + + event: message_stop + data: {"type":"message_stop" } + + headers: + CF-RAY: + - 9c72d9568a0f25dc-EWR + Cache-Control: + - no-cache + Connection: + - keep-alive + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Sun, 01 Feb 2026 16:33:11 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 455ea6be-bd92-4199-83ec-0c6b39c5c169 + anthropic-ratelimit-input-tokens-limit: + - '30000' + anthropic-ratelimit-input-tokens-remaining: + - '30000' + anthropic-ratelimit-input-tokens-reset: + - '2026-02-01T16:33:10Z' + anthropic-ratelimit-output-tokens-limit: + - '8000' + anthropic-ratelimit-output-tokens-remaining: + - '8000' + anthropic-ratelimit-output-tokens-reset: + - '2026-02-01T16:33:10Z' + anthropic-ratelimit-requests-limit: + - '50' + anthropic-ratelimit-requests-remaining: + - '49' + anthropic-ratelimit-requests-reset: + - '2026-02-01T16:33:11Z' + anthropic-ratelimit-tokens-limit: + - '38000' + anthropic-ratelimit-tokens-remaining: + - '38000' + anthropic-ratelimit-tokens-reset: + - '2026-02-01T16:33:10Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CXhfsBUNK7FvbBZy9iz9q + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - '1154' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_streaming_iteration.yaml b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_streaming_iteration.yaml new file mode 100644 index 0000000000..778e4c6ab2 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_streaming_iteration.yaml @@ -0,0 +1,145 @@ +interactions: +- request: + body: |- + { + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Say hi." + } + ], + "model": "claude-sonnet-4-20250514", + "stream": true + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '116' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 0.75.0 + x-api-key: + - test_anthropic_api_key + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.75.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.9.6 + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |+ + event: message_start + data: {"type":"message_start","message":{"model":"claude-sonnet-4-20250514","id":"msg_01Gx7i9fVFvFzwgqaXjX1aY8","type":"message","role":"assistant","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard"}} } + + event: content_block_start + data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} } + + event: ping + data: {"type": "ping"} + + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hi"} } + + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"!"} } + + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" How"} } + + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" are you doing today?"} } + + event: content_block_stop + data: {"type":"content_block_stop","index":0 } + + event: message_delta + data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":10,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":11} } + + event: message_stop + data: {"type":"message_stop" } + + headers: + CF-RAY: + - 9c72d95f4915f791-EWR + Cache-Control: + - no-cache + Connection: + - keep-alive + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Sun, 01 Feb 2026 16:33:12 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 455ea6be-bd92-4199-83ec-0c6b39c5c169 + anthropic-ratelimit-input-tokens-limit: + - '30000' + anthropic-ratelimit-input-tokens-remaining: + - '30000' + anthropic-ratelimit-input-tokens-reset: + - '2026-02-01T16:33:11Z' + anthropic-ratelimit-output-tokens-limit: + - '8000' + anthropic-ratelimit-output-tokens-remaining: + - '8000' + anthropic-ratelimit-output-tokens-reset: + - '2026-02-01T16:33:11Z' + anthropic-ratelimit-requests-limit: + - '50' + anthropic-ratelimit-requests-remaining: + - '49' + anthropic-ratelimit-requests-reset: + - '2026-02-01T16:33:12Z' + anthropic-ratelimit-tokens-limit: + - '38000' + anthropic-ratelimit-tokens-remaining: + - '38000' + anthropic-ratelimit-tokens-reset: + - '2026-02-01T16:33:11Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CXhfsHR44t4VmLrVSHw7n + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - '1188' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_token_usage.yaml b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_token_usage.yaml new file mode 100644 index 0000000000..4269e7f324 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_token_usage.yaml @@ -0,0 +1,139 @@ +interactions: +- request: + body: |- + { + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Count to 5." + } + ], + "model": "claude-sonnet-4-20250514" + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '106' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 0.75.0 + x-api-key: + - test_anthropic_api_key + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.75.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.9.6 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "model": "claude-sonnet-4-20250514", + "id": "msg_01KAqSYfNUEygm2fAUn6Aoei", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "1\n2\n3\n4\n5" + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 12, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": { + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 13, + "service_tier": "standard" + } + } + headers: + CF-RAY: + - 9c72d9386b87be82-EWR + Connection: + - keep-alive + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - application/json + Date: + - Sun, 01 Feb 2026 16:33:06 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 455ea6be-bd92-4199-83ec-0c6b39c5c169 + anthropic-ratelimit-input-tokens-limit: + - '30000' + anthropic-ratelimit-input-tokens-remaining: + - '30000' + anthropic-ratelimit-input-tokens-reset: + - '2026-02-01T16:33:06Z' + anthropic-ratelimit-output-tokens-limit: + - '8000' + anthropic-ratelimit-output-tokens-remaining: + - '8000' + anthropic-ratelimit-output-tokens-reset: + - '2026-02-01T16:33:06Z' + anthropic-ratelimit-requests-limit: + - '50' + anthropic-ratelimit-requests-remaining: + - '49' + anthropic-ratelimit-requests-reset: + - '2026-02-01T16:33:06Z' + anthropic-ratelimit-tokens-limit: + - '38000' + anthropic-ratelimit-tokens-remaining: + - '38000' + anthropic-ratelimit-tokens-reset: + - '2026-02-01T16:33:06Z' + cf-cache-status: + - DYNAMIC + content-length: + - '417' + request-id: + - req_011CXhfrpwqw8NoC8t7gaDpC + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - '1463' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_with_all_params.yaml b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_with_all_params.yaml new file mode 100644 index 0000000000..7ebd714698 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_create_with_all_params.yaml @@ -0,0 +1,145 @@ +interactions: +- request: + body: |- + { + "max_tokens": 50, + "messages": [ + { + "role": "user", + "content": "Say hello." + } + ], + "model": "claude-sonnet-4-20250514", + "stop_sequences": [ + "STOP" + ], + "temperature": 0.7, + "top_k": 40, + "top_p": 0.9 + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '171' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 0.75.0 + x-api-key: + - test_anthropic_api_key + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.75.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.9.6 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "model": "claude-sonnet-4-20250514", + "id": "msg_01NbfqvRwE8YGETpAgiWufrT", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "Hello! How are you doing today?" + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 10, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": { + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 11, + "service_tier": "standard" + } + } + headers: + CF-RAY: + - 9c72d92efbe81ea7-EWR + Connection: + - keep-alive + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - application/json + Date: + - Sun, 01 Feb 2026 16:33:05 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 455ea6be-bd92-4199-83ec-0c6b39c5c169 + anthropic-ratelimit-input-tokens-limit: + - '30000' + anthropic-ratelimit-input-tokens-remaining: + - '30000' + anthropic-ratelimit-input-tokens-reset: + - '2026-02-01T16:33:04Z' + anthropic-ratelimit-output-tokens-limit: + - '8000' + anthropic-ratelimit-output-tokens-remaining: + - '8000' + anthropic-ratelimit-output-tokens-reset: + - '2026-02-01T16:33:05Z' + anthropic-ratelimit-requests-limit: + - '50' + anthropic-ratelimit-requests-remaining: + - '49' + anthropic-ratelimit-requests-reset: + - '2026-02-01T16:33:05Z' + anthropic-ratelimit-tokens-limit: + - '38000' + anthropic-ratelimit-tokens-remaining: + - '38000' + anthropic-ratelimit-tokens-reset: + - '2026-02-01T16:33:04Z' + cf-cache-status: + - DYNAMIC + content-length: + - '435' + request-id: + - req_011CXhfriMxYAvFnjiVTEw1i + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - '1268' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_stream_basic.yaml b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_stream_basic.yaml new file mode 100644 index 0000000000..8c9d03d833 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_stream_basic.yaml @@ -0,0 +1,140 @@ +interactions: +- request: + body: |- + { + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Say hello in one word." + } + ], + "model": "claude-sonnet-4-20250514", + "stream": true + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '131' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 0.75.0 + x-api-key: + - test_anthropic_api_key + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-helper-method: + - stream + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.75.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.9.6 + x-stainless-stream-helper: + - messages + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |+ + event: message_start + data: {"type":"message_start","message":{"model":"claude-sonnet-4-20250514","id":"msg_0162DFdxGj47acpXbV8kNbuJ","type":"message","role":"assistant","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":13,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":2,"service_tier":"standard"}} } + + event: content_block_start + data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} } + + event: ping + data: {"type": "ping"} + + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello!"} } + + event: content_block_stop + data: {"type":"content_block_stop","index":0 } + + event: message_delta + data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":13,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":5} } + + event: message_stop + data: {"type":"message_stop" } + + headers: + CF-RAY: + - 9c72d971ae23dd82-EWR + Cache-Control: + - no-cache + Connection: + - keep-alive + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Sun, 01 Feb 2026 16:33:15 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 455ea6be-bd92-4199-83ec-0c6b39c5c169 + anthropic-ratelimit-input-tokens-limit: + - '30000' + anthropic-ratelimit-input-tokens-remaining: + - '30000' + anthropic-ratelimit-input-tokens-reset: + - '2026-02-01T16:33:14Z' + anthropic-ratelimit-output-tokens-limit: + - '8000' + anthropic-ratelimit-output-tokens-remaining: + - '8000' + anthropic-ratelimit-output-tokens-reset: + - '2026-02-01T16:33:14Z' + anthropic-ratelimit-requests-limit: + - '50' + anthropic-ratelimit-requests-remaining: + - '49' + anthropic-ratelimit-requests-reset: + - '2026-02-01T16:33:15Z' + anthropic-ratelimit-tokens-limit: + - '38000' + anthropic-ratelimit-tokens-remaining: + - '38000' + anthropic-ratelimit-tokens-reset: + - '2026-02-01T16:33:14Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CXhfsVzqovX74veR7mZ8B + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - '1154' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_stream_token_usage.yaml b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_stream_token_usage.yaml new file mode 100644 index 0000000000..6630b51147 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_stream_token_usage.yaml @@ -0,0 +1,143 @@ +interactions: +- request: + body: |- + { + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Count to 3." + } + ], + "model": "claude-sonnet-4-20250514", + "stream": true + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '120' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 0.75.0 + x-api-key: + - test_anthropic_api_key + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-helper-method: + - stream + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.75.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.9.6 + x-stainless-stream-helper: + - messages + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |+ + event: message_start + data: {"type":"message_start","message":{"model":"claude-sonnet-4-20250514","id":"msg_01Yabq2jNeX59V8PNcyHCvBe","type":"message","role":"assistant","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":12,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard"}} } + + event: content_block_start + data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} } + + event: ping + data: {"type": "ping"} + + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"1,"} } + + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" 2, 3"} } + + event: content_block_stop + data: {"type":"content_block_stop","index":0 } + + event: message_delta + data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":12,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":11} } + + event: message_stop + data: {"type":"message_stop" } + + headers: + CF-RAY: + - 9c72d985adc4428f-EWR + Cache-Control: + - no-cache + Connection: + - keep-alive + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Sun, 01 Feb 2026 16:33:19 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 455ea6be-bd92-4199-83ec-0c6b39c5c169 + anthropic-ratelimit-input-tokens-limit: + - '30000' + anthropic-ratelimit-input-tokens-remaining: + - '30000' + anthropic-ratelimit-input-tokens-reset: + - '2026-02-01T16:33:17Z' + anthropic-ratelimit-output-tokens-limit: + - '8000' + anthropic-ratelimit-output-tokens-remaining: + - '8000' + anthropic-ratelimit-output-tokens-reset: + - '2026-02-01T16:33:17Z' + anthropic-ratelimit-requests-limit: + - '50' + anthropic-ratelimit-requests-remaining: + - '49' + anthropic-ratelimit-requests-reset: + - '2026-02-01T16:33:19Z' + anthropic-ratelimit-tokens-limit: + - '38000' + anthropic-ratelimit-tokens-remaining: + - '38000' + anthropic-ratelimit-tokens-reset: + - '2026-02-01T16:33:17Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CXhfsjgcXuKCQdgsaYFv7 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - '1385' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_stream_with_params.yaml b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_stream_with_params.yaml new file mode 100644 index 0000000000..7258f8f801 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/cassettes/test_async_messages_stream_with_params.yaml @@ -0,0 +1,155 @@ +interactions: +- request: + body: |- + { + "max_tokens": 50, + "messages": [ + { + "role": "user", + "content": "Say hi." + } + ], + "model": "claude-sonnet-4-20250514", + "temperature": 0.7, + "top_k": 40, + "top_p": 0.9, + "stream": true + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '156' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 0.75.0 + x-api-key: + - test_anthropic_api_key + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-helper-method: + - stream + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.75.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.9.6 + x-stainless-stream-helper: + - messages + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |+ + event: message_start + data: {"type":"message_start","message":{"model":"claude-sonnet-4-20250514","id":"msg_01KchPeMpi62bUDMuHVaFZ4c","type":"message","role":"assistant","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard"}} } + + event: content_block_start + data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} } + + event: ping + data: {"type": "ping"} + + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hi"} } + + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"!"}} + + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" How"} } + + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" are"} } + + event: content_block_delta + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" you doing today?"} } + + event: content_block_stop + data: {"type":"content_block_stop","index":0 } + + event: message_delta + data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":10,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":11} } + + event: message_stop + data: {"type":"message_stop" } + + headers: + CF-RAY: + - 9c72d97a980eadca-EWR + Cache-Control: + - no-cache + Connection: + - keep-alive + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Sun, 01 Feb 2026 16:33:17 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 455ea6be-bd92-4199-83ec-0c6b39c5c169 + anthropic-ratelimit-input-tokens-limit: + - '30000' + anthropic-ratelimit-input-tokens-remaining: + - '30000' + anthropic-ratelimit-input-tokens-reset: + - '2026-02-01T16:33:16Z' + anthropic-ratelimit-output-tokens-limit: + - '8000' + anthropic-ratelimit-output-tokens-remaining: + - '8000' + anthropic-ratelimit-output-tokens-reset: + - '2026-02-01T16:33:16Z' + anthropic-ratelimit-requests-limit: + - '50' + anthropic-ratelimit-requests-remaining: + - '49' + anthropic-ratelimit-requests-reset: + - '2026-02-01T16:33:17Z' + anthropic-ratelimit-tokens-limit: + - '38000' + anthropic-ratelimit-tokens-remaining: + - '38000' + anthropic-ratelimit-tokens-reset: + - '2026-02-01T16:33:16Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CXhfsc6DLELMwZPK98mem + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - '1222' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/conftest.py b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/conftest.py index 9f660d1fd4..003c2ab26f 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/conftest.py +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/conftest.py @@ -20,7 +20,7 @@ import pytest import yaml -from anthropic import Anthropic +from anthropic import Anthropic, AsyncAnthropic from opentelemetry.instrumentation.anthropic import AnthropicInstrumentor from opentelemetry.sdk._logs import LoggerProvider @@ -99,6 +99,12 @@ def anthropic_client(): return Anthropic() +@pytest.fixture +def async_anthropic_client(): + """Create and return an AsyncAnthropic client.""" + return AsyncAnthropic() + + @pytest.fixture(scope="module") def vcr_config(): """Configure VCR for recording/replaying HTTP interactions.""" diff --git a/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/test_async_messages.py b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/test_async_messages.py new file mode 100644 index 0000000000..94f883077d --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-anthropic/tests/test_async_messages.py @@ -0,0 +1,554 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for async AsyncMessages.create and AsyncMessages.stream instrumentation.""" + +import pytest +from anthropic import APIConnectionError, AsyncAnthropic, NotFoundError + +from opentelemetry.semconv._incubating.attributes import ( + error_attributes as ErrorAttributes, +) +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) +from opentelemetry.semconv._incubating.attributes import ( + server_attributes as ServerAttributes, +) + + +def normalize_stop_reason(stop_reason): + """Map Anthropic stop reasons to GenAI semconv values.""" + return { + "end_turn": "stop", + "stop_sequence": "stop", + "max_tokens": "length", + "tool_use": "tool_calls", + }.get(stop_reason, stop_reason) + + +def expected_input_tokens(usage): + """Compute semconv input tokens from Anthropic usage.""" + base = getattr(usage, "input_tokens", 0) or 0 + cache_creation = getattr(usage, "cache_creation_input_tokens", 0) or 0 + cache_read = getattr(usage, "cache_read_input_tokens", 0) or 0 + return base + cache_creation + cache_read + + +def assert_span_attributes( # pylint: disable=too-many-arguments + span, + request_model, + response_id=None, + response_model=None, + input_tokens=None, + output_tokens=None, + finish_reasons=None, + operation_name="chat", + server_address="api.anthropic.com", +): + """Assert that a span has the expected attributes.""" + assert span.name == f"{operation_name} {request_model}" + assert ( + operation_name + == span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + ) + assert ( + GenAIAttributes.GenAiSystemValues.ANTHROPIC.value + == span.attributes[GenAIAttributes.GEN_AI_SYSTEM] + ) + assert ( + request_model == span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] + ) + assert server_address == span.attributes[ServerAttributes.SERVER_ADDRESS] + + if response_id is not None: + assert ( + response_id == span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] + ) + + if response_model is not None: + assert ( + response_model + == span.attributes[GenAIAttributes.GEN_AI_RESPONSE_MODEL] + ) + + if input_tokens is not None: + assert ( + input_tokens + == span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] + ) + + if output_tokens is not None: + assert ( + output_tokens + == span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] + ) + + if finish_reasons is not None: + # OpenTelemetry converts lists to tuples when storing as attributes + assert ( + tuple(finish_reasons) + == span.attributes[GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS] + ) + + +# ============================================================================= +# Tests for AsyncMessages.create() method +# ============================================================================= + + +@pytest.mark.vcr() +@pytest.mark.asyncio +async def test_async_messages_create_basic( + span_exporter, async_anthropic_client, instrument_no_content +): + """Test basic async message creation produces correct span.""" + model = "claude-sonnet-4-20250514" + messages = [{"role": "user", "content": "Say hello in one word."}] + + response = await async_anthropic_client.messages.create( + model=model, + max_tokens=100, + messages=messages, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + + assert_span_attributes( + spans[0], + request_model=model, + response_id=response.id, + response_model=response.model, + input_tokens=expected_input_tokens(response.usage), + output_tokens=response.usage.output_tokens, + finish_reasons=[normalize_stop_reason(response.stop_reason)], + ) + + +@pytest.mark.vcr() +@pytest.mark.asyncio +async def test_async_messages_create_with_all_params( + span_exporter, async_anthropic_client, instrument_no_content +): + """Test async message creation with all optional parameters.""" + model = "claude-sonnet-4-20250514" + messages = [{"role": "user", "content": "Say hello."}] + + await async_anthropic_client.messages.create( + model=model, + max_tokens=50, + messages=messages, + temperature=0.7, + top_p=0.9, + top_k=40, + stop_sequences=["STOP"], + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_MAX_TOKENS] == 50 + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_TOP_P] == 0.9 + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_TOP_K] == 40 + # OpenTelemetry converts lists to tuples when storing as attributes + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_STOP_SEQUENCES] == ( + "STOP", + ) + + +@pytest.mark.vcr() +@pytest.mark.asyncio +async def test_async_messages_create_token_usage( + span_exporter, async_anthropic_client, instrument_no_content +): + """Test that token usage is captured correctly for async create.""" + model = "claude-sonnet-4-20250514" + messages = [{"role": "user", "content": "Count to 5."}] + + response = await async_anthropic_client.messages.create( + model=model, + max_tokens=100, + messages=messages, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS in span.attributes + assert GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS in span.attributes + assert span.attributes[ + GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS + ] == expected_input_tokens(response.usage) + assert ( + span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] + == response.usage.output_tokens + ) + assert span.attributes["gen_ai.usage.cache_creation.input_tokens"] == ( + response.usage.cache_creation_input_tokens or 0 + ) + assert span.attributes["gen_ai.usage.cache_read.input_tokens"] == ( + response.usage.cache_read_input_tokens or 0 + ) + + +@pytest.mark.vcr() +@pytest.mark.asyncio +async def test_async_messages_create_stop_reason( + span_exporter, async_anthropic_client, instrument_no_content +): + """Test that stop reason is captured as finish_reasons array for async create.""" + model = "claude-sonnet-4-20250514" + messages = [{"role": "user", "content": "Say hi."}] + + response = await async_anthropic_client.messages.create( + model=model, + max_tokens=100, + messages=messages, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + # Anthropic's stop_reason should be wrapped in a tuple (OTel converts lists) + assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS] == ( + normalize_stop_reason(response.stop_reason), + ) + + +@pytest.mark.asyncio +async def test_async_messages_create_connection_error( + span_exporter, instrument_no_content +): + """Test that connection errors are handled correctly for async create.""" + model = "claude-sonnet-4-20250514" + messages = [{"role": "user", "content": "Hello"}] + + # Create client with invalid endpoint + client = AsyncAnthropic(base_url="http://localhost:9999") + + with pytest.raises(APIConnectionError): + await client.messages.create( + model=model, + max_tokens=100, + messages=messages, + timeout=0.1, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == model + assert ErrorAttributes.ERROR_TYPE in span.attributes + assert "APIConnectionError" in span.attributes[ErrorAttributes.ERROR_TYPE] + + +@pytest.mark.vcr() +@pytest.mark.asyncio +async def test_async_messages_create_api_error( + span_exporter, async_anthropic_client, instrument_no_content +): + """Test that API errors (e.g., invalid model) are handled correctly for async.""" + model = "invalid-model-name" + messages = [{"role": "user", "content": "Hello"}] + + with pytest.raises(NotFoundError): + await async_anthropic_client.messages.create( + model=model, + max_tokens=100, + messages=messages, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == model + assert ErrorAttributes.ERROR_TYPE in span.attributes + assert "NotFoundError" in span.attributes[ErrorAttributes.ERROR_TYPE] + + +@pytest.mark.vcr() +@pytest.mark.asyncio +async def test_async_messages_create_streaming( # pylint: disable=too-many-locals + span_exporter, async_anthropic_client, instrument_no_content +): + """Test async streaming message creation produces correct span.""" + model = "claude-sonnet-4-20250514" + messages = [{"role": "user", "content": "Say hello in one word."}] + + # Collect response data from stream + response_text = "" + response_id = None + response_model = None + stop_reason = None + input_tokens = None + output_tokens = None + + # Note: AsyncMessages.create() is a coroutine that must be awaited first, + # then you can use async with on the result (unlike sync which returns directly) + stream = await async_anthropic_client.messages.create( + model=model, + max_tokens=100, + messages=messages, + stream=True, + ) + async with stream: + async for chunk in stream: + # Extract data from chunks for assertion + if chunk.type == "message_start": + message = getattr(chunk, "message", None) + if message: + response_id = getattr(message, "id", None) + response_model = getattr(message, "model", None) + usage = getattr(message, "usage", None) + if usage: + input_tokens = expected_input_tokens(usage) + elif chunk.type == "content_block_delta": + delta = getattr(chunk, "delta", None) + if delta and hasattr(delta, "text"): + response_text += delta.text + elif chunk.type == "message_delta": + delta = getattr(chunk, "delta", None) + if delta: + stop_reason = getattr(delta, "stop_reason", None) + usage = getattr(chunk, "usage", None) + if usage: + output_tokens = getattr(usage, "output_tokens", None) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + + assert_span_attributes( + spans[0], + request_model=model, + response_id=response_id, + response_model=response_model, + input_tokens=input_tokens, + output_tokens=output_tokens, + finish_reasons=[normalize_stop_reason(stop_reason)] + if stop_reason + else None, + ) + + +@pytest.mark.vcr() +@pytest.mark.asyncio +async def test_async_messages_create_streaming_iteration( + span_exporter, async_anthropic_client, instrument_no_content +): + """Test async streaming with direct iteration (without context manager).""" + model = "claude-sonnet-4-20250514" + messages = [{"role": "user", "content": "Say hi."}] + + stream = await async_anthropic_client.messages.create( + model=model, + max_tokens=100, + messages=messages, + stream=True, + ) + + # Consume the stream by iterating + chunks = [] + async for chunk in stream: + chunks.append(chunk) + assert len(chunks) > 0 + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == model + # Verify span has response attributes from streaming + assert GenAIAttributes.GEN_AI_RESPONSE_ID in span.attributes + assert GenAIAttributes.GEN_AI_RESPONSE_MODEL in span.attributes + + +@pytest.mark.asyncio +async def test_async_messages_create_streaming_connection_error( + span_exporter, instrument_no_content +): + """Test that connection errors during async streaming are handled correctly.""" + model = "claude-sonnet-4-20250514" + messages = [{"role": "user", "content": "Hello"}] + + # Create client with invalid endpoint + client = AsyncAnthropic(base_url="http://localhost:9999") + + with pytest.raises(APIConnectionError): + await client.messages.create( + model=model, + max_tokens=100, + messages=messages, + stream=True, + timeout=0.1, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == model + assert ErrorAttributes.ERROR_TYPE in span.attributes + assert "APIConnectionError" in span.attributes[ErrorAttributes.ERROR_TYPE] + + +# ============================================================================= +# Tests for AsyncMessages.stream() method +# ============================================================================= + + +@pytest.mark.vcr() +@pytest.mark.asyncio +async def test_async_messages_stream_basic( + span_exporter, async_anthropic_client, instrument_no_content +): + """Test AsyncMessages.stream() produces correct span with async context manager.""" + model = "claude-sonnet-4-20250514" + messages = [{"role": "user", "content": "Say hello in one word."}] + + async with async_anthropic_client.messages.stream( + model=model, + max_tokens=100, + messages=messages, + ) as stream: + # Consume the stream using text_stream + text_parts = [] + async for text in stream.text_stream: + text_parts.append(text) + response_text = "".join(text_parts) + # Get the final message for assertions + final_message = await stream.get_final_message() + + assert response_text # Should have some text + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + + assert_span_attributes( + spans[0], + request_model=model, + response_id=final_message.id, + response_model=final_message.model, + input_tokens=expected_input_tokens(final_message.usage), + output_tokens=final_message.usage.output_tokens, + finish_reasons=[normalize_stop_reason(final_message.stop_reason)], + ) + + +@pytest.mark.vcr() +@pytest.mark.asyncio +async def test_async_messages_stream_with_params( + span_exporter, async_anthropic_client, instrument_no_content +): + """Test AsyncMessages.stream() with additional parameters.""" + model = "claude-sonnet-4-20250514" + messages = [{"role": "user", "content": "Say hi."}] + + async with async_anthropic_client.messages.stream( + model=model, + max_tokens=50, + messages=messages, + temperature=0.7, + top_p=0.9, + top_k=40, + ) as stream: + # Consume the stream + async for _ in stream.text_stream: + pass + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == model + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_MAX_TOKENS] == 50 + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_TOP_P] == 0.9 + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_TOP_K] == 40 + + +@pytest.mark.vcr() +@pytest.mark.asyncio +async def test_async_messages_stream_token_usage( + span_exporter, async_anthropic_client, instrument_no_content +): + """Test that AsyncMessages.stream() captures token usage correctly.""" + model = "claude-sonnet-4-20250514" + messages = [{"role": "user", "content": "Count to 3."}] + + async with async_anthropic_client.messages.stream( + model=model, + max_tokens=100, + messages=messages, + ) as stream: + async for _ in stream.text_stream: + pass + final_message = await stream.get_final_message() + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS in span.attributes + assert GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS in span.attributes + assert span.attributes[ + GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS + ] == expected_input_tokens(final_message.usage) + assert ( + span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] + == final_message.usage.output_tokens + ) + assert span.attributes["gen_ai.usage.cache_creation.input_tokens"] == ( + final_message.usage.cache_creation_input_tokens or 0 + ) + assert span.attributes["gen_ai.usage.cache_read.input_tokens"] == ( + final_message.usage.cache_read_input_tokens or 0 + ) + + +@pytest.mark.asyncio +async def test_async_messages_stream_connection_error( + span_exporter, instrument_no_content +): + """Test that connection errors in AsyncMessages.stream() are handled correctly.""" + model = "claude-sonnet-4-20250514" + messages = [{"role": "user", "content": "Hello"}] + + # Create client with invalid endpoint + client = AsyncAnthropic(base_url="http://localhost:9999") + + with pytest.raises(APIConnectionError): + # pylint: disable=not-async-context-manager + async with client.messages.stream( + model=model, + max_tokens=100, + messages=messages, + timeout=0.1, + ) as stream: + # Try to consume the stream + async for _ in stream.text_stream: + pass + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == model + assert ErrorAttributes.ERROR_TYPE in span.attributes + assert "APIConnectionError" in span.attributes[ErrorAttributes.ERROR_TYPE]