diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/CHANGELOG.md b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/CHANGELOG.md index 87dc9461e7..313b6ce7b0 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/CHANGELOG.md +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/CHANGELOG.md @@ -28,6 +28,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add strongly typed Responses API extractors with validation and content extraction improvements ([#4337](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4337)) +- Add instrumentation for OpenAI Responses API `create` + ([#4474](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4474)) ## Version 2.3b0 (2025-12-24) diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py index f001812def..f74a4796a2 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py @@ -40,6 +40,7 @@ --- """ +from importlib import import_module from typing import Collection from wrapt import wrap_function_wrapper @@ -70,6 +71,9 @@ chat_completions_create_v_old, embeddings_create, ) +from .patch_responses import ( + responses_create, +) class OpenAIInstrumentor(BaseInstrumentor): @@ -159,6 +163,19 @@ def _instrument(self, **kwargs): ), ) + responses_module = _get_responses_module() + # Responses instrumentation is intentionally limited to the latest + # experimental semconv path. Unlike chat completions, we do not carry + # a second legacy wrapper here; the current implementation is built on + # the inference handler lifecycle and would need a separate old-path + # implementation to support legacy semconv mode. + if responses_module is not None and latest_experimental_enabled: + wrap_function_wrapper( + "openai.resources.responses.responses", + "Responses.create", + responses_create(handler, content_mode), + ) + def _uninstrument(self, **kwargs): import openai # pylint: disable=import-outside-toplevel # noqa: PLC0415 @@ -166,3 +183,13 @@ def _uninstrument(self, **kwargs): unwrap(openai.resources.chat.completions.AsyncCompletions, "create") unwrap(openai.resources.embeddings.Embeddings, "create") unwrap(openai.resources.embeddings.AsyncEmbeddings, "create") + responses_module = _get_responses_module() + if responses_module is not None: + unwrap(responses_module.Responses, "create") + + +def _get_responses_module(): + try: + return import_module("openai.resources.responses.responses") + except ImportError: + return None diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch_responses.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch_responses.py new file mode 100644 index 0000000000..65ef3b6597 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch_responses.py @@ -0,0 +1,69 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from opentelemetry.util.genai.handler import TelemetryHandler +from opentelemetry.util.genai.types import ContentCapturingMode, Error + +from .response_extractors import ( + _apply_request_attributes, + _get_inference_creation_kwargs, + _set_invocation_response_attributes, +) +from .response_wrappers import ResponseStreamWrapper +from .utils import is_streaming + + +def responses_create( + handler: TelemetryHandler, + content_capturing_mode: ContentCapturingMode, +): + """Wrap the `create` method of the `Responses` class to trace it.""" + + capture_content = content_capturing_mode != ContentCapturingMode.NO_CONTENT + + def traced_method(wrapped, instance, args, kwargs): + invocation = handler.start_inference( + **_get_inference_creation_kwargs(kwargs, instance) + ) + _apply_request_attributes(invocation, kwargs, capture_content) + + try: + result = wrapped(*args, **kwargs) + parsed_result = _get_response_stream_result(result) + + if is_streaming(kwargs): + return ResponseStreamWrapper( + parsed_result, + invocation, + capture_content, + ) + + _set_invocation_response_attributes( + invocation, parsed_result, capture_content + ) + invocation.stop() + return result + except Exception as error: + invocation.fail(Error(type=type(error), message=str(error))) + raise + + return traced_method + + +def _get_response_stream_result(result): + if hasattr(result, "parse"): + return result.parse() + return result diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/response_extractors.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/response_extractors.py index 2d5e8702b1..bf4f3feb7e 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/response_extractors.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/response_extractors.py @@ -14,16 +14,29 @@ from __future__ import annotations +import json import logging from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, List, Optional, TypeVar, Union -from pydantic import BaseModel, Field, StrictInt, StrictStr, ValidationError +from pydantic import ( + BaseModel, + Field, + StrictFloat, + StrictInt, + StrictStr, + ValidationError, +) +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) from opentelemetry.semconv._incubating.attributes import ( openai_attributes as OpenAIAttributes, ) +from .utils import get_server_address_and_port + _PYDANTIC_V2 = hasattr(BaseModel, "model_validate") if _PYDANTIC_V2: @@ -42,13 +55,21 @@ try: from opentelemetry.util.genai.types import ( InputMessage, + LLMInvocation, OutputMessage, + Reasoning, Text, ) + from opentelemetry.util.genai.types import ( + ToolCallRequest as ToolCall, + ) except ImportError: InputMessage = None + LLMInvocation = None OutputMessage = None + Reasoning = None Text = None + ToolCall = None GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS = "gen_ai.usage.cache_read.input_tokens" GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS = ( @@ -91,7 +112,12 @@ class _ResponseInputItemModel(_ExtractorModel): class _ResponsesRequestModel(_ExtractorModel): instructions: Optional[StrictStr] = None input: Optional[Union[StrictStr, List[_ResponseInputItemModel]]] = None + max_output_tokens: Optional[StrictInt] = None + model: Optional[StrictStr] = None + service_tier: Optional[StrictStr] = None + temperature: Optional[StrictFloat] = None text: Optional[_ResponseTextConfigModel] = None + top_p: Optional[StrictFloat] = None class _ResponseOutputContentModel(_ExtractorModel): @@ -104,7 +130,12 @@ class _ResponseOutputItemModel(_ExtractorModel): type: Optional[StrictStr] = None role: Optional[StrictStr] = None status: Optional[StrictStr] = None + id: Optional[StrictStr] = None + call_id: Optional[StrictStr] = None + name: Optional[StrictStr] = None + arguments: Optional[StrictStr] = None content: List[_ResponseOutputContentModel] = Field(default_factory=list) + summary: List[_ResponseOutputContentModel] = Field(default_factory=list) class _UsageDetailsModel(_ExtractorModel): @@ -264,6 +295,32 @@ def _extract_output_parts( return parts +def _parse_tool_call_arguments(arguments: str | None) -> object: + if arguments is None: + return None + + try: + return json.loads(arguments) + except (TypeError, ValueError): + return arguments + + +def _extract_reasoning_parts( + item: _ResponseOutputItemModel, +) -> list["Reasoning"]: + if Reasoning is None: + return [] + + parts: list[Reasoning] = [] + for block in item.summary: + if block.text is not None: + parts.append(Reasoning(content=block.text)) + for block in item.content: + if block.type == "reasoning_text" and block.text is not None: + parts.append(Reasoning(content=block.text)) + return parts + + def _finish_reason_from_status(status: str | None) -> str | None: # Responses API output items expose lifecycle statuses rather than finish # reasons. We map the normal terminal state to the GenAI "stop" reason, @@ -284,19 +341,57 @@ def _extract_output_messages_from_model( messages: list[OutputMessage] = [] for item in result.output: - if item.type != "message": - continue - finish_reason = _finish_reason_from_status(item.status) - if finish_reason is None: + if item.type == "message": + finish_reason = _finish_reason_from_status(item.status) + if finish_reason is None: + continue + + messages.append( + OutputMessage( + role=item.role if item.role is not None else "assistant", + parts=_extract_output_parts(item.content), + finish_reason=finish_reason, + ) + ) continue - messages.append( - OutputMessage( - role=item.role if item.role is not None else "assistant", - parts=_extract_output_parts(item.content), - finish_reason=finish_reason, + if item.type == "function_call": + if ToolCall is None or item.name is None: + continue + if item.status not in {"completed", "incomplete"}: + continue + + messages.append( + OutputMessage( + role="assistant", + parts=[ + ToolCall( + id=item.call_id if item.call_id else item.id, + name=item.name, + arguments=_parse_tool_call_arguments( + item.arguments + ), + ) + ], + finish_reason="tool_calls", + ) ) - ) + continue + + if item.type == "reasoning": + finish_reason = _finish_reason_from_status(item.status) + if finish_reason is None: + continue + + parts = _extract_reasoning_parts(item) + if parts: + messages.append( + OutputMessage( + role="assistant", + parts=parts, + finish_reason=finish_reason, + ) + ) return messages @@ -317,12 +412,19 @@ def _extract_finish_reasons_from_model( ) -> list[str]: finish_reasons: list[str] = [] for item in result.output: + if item.type == "function_call" and item.status in { + "completed", + "incomplete", + }: + finish_reasons.append("tool_calls") + continue + if item.type != "message": continue finish_reason = _finish_reason_from_status(item.status) if finish_reason is not None: finish_reasons.append(finish_reason) - return finish_reasons + return list(dict.fromkeys(finish_reasons)) def _extract_finish_reasons( @@ -347,9 +449,68 @@ def _extract_output_type(kwargs: Mapping[str, object]) -> str | None: return request.text.format.type -def _set_invocation_usage_attributes( - invocation: "LLMInvocation", usage: _UsageModel +def _extract_request_service_tier( + kwargs: Mapping[str, object], +) -> str | None: + request = _validate_request_kwargs(kwargs) + if request is None: + return None + + service_tier = request.service_tier + if service_tier in (None, "auto"): + return None + + return service_tier + + +def _get_inference_creation_kwargs( + kwargs: Mapping[str, object], + client_instance: object, +) -> dict[str, object]: + request = _validate_request_kwargs(kwargs) + request_model = request.model if request is not None else None + address, port = get_server_address_and_port(client_instance) + + creation_kwargs: dict[str, object] = { + "provider": GenAIAttributes.GenAiProviderNameValues.OPENAI.value, + } + if request_model is not None: + creation_kwargs["request_model"] = request_model + if address is not None: + creation_kwargs["server_address"] = address + if port is not None: + creation_kwargs["server_port"] = port + return creation_kwargs + + +def _apply_request_attributes( + invocation, + kwargs: Mapping[str, object], + capture_content: bool, ) -> None: + request = _validate_request_kwargs(kwargs) + + if request is not None: + invocation.temperature = request.temperature + invocation.top_p = request.top_p + invocation.max_tokens = request.max_output_tokens + + request_service_tier = _extract_request_service_tier(kwargs) + if request_service_tier is not None: + invocation.attributes[OpenAIAttributes.OPENAI_REQUEST_SERVICE_TIER] = ( + request_service_tier + ) + + output_type = _extract_output_type(kwargs) + if output_type is not None: + invocation.attributes[GenAIAttributes.GEN_AI_OUTPUT_TYPE] = output_type + + if capture_content: + invocation.system_instruction = _extract_system_instruction(kwargs) + invocation.input_messages = _extract_input_messages(kwargs) + + +def _set_invocation_usage_attributes(invocation, usage: _UsageModel) -> None: if usage.input_tokens is not None: invocation.input_tokens = usage.input_tokens else: @@ -380,7 +541,7 @@ def _set_invocation_usage_attributes( def _set_invocation_response_attributes( - invocation: "LLMInvocation", + invocation, result: object | None, capture_content: bool, ) -> None: diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/response_wrappers.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/response_wrappers.py index 402e5c84c2..e13cc6a400 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/response_wrappers.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/response_wrappers.py @@ -7,42 +7,7 @@ from types import TracebackType from typing import TYPE_CHECKING, Callable, Generator, Generic, TypeVar -from opentelemetry.util.genai.handler import TelemetryHandler -from opentelemetry.util.genai.types import ( - Error, - LLMInvocation, # pylint: disable=no-name-in-module # TODO: migrate to InferenceInvocation -) - -# OpenAI Responses internals are version-gated (added in openai>=1.66.0), so -# pylint may not resolve them in all lint environments even though we guard -# runtime usage with ImportError fallbacks below. -try: - from openai.lib.streaming.responses._events import ( # pylint: disable=no-name-in-module - ResponseCompletedEvent, - ) - from openai.types.responses import ( # pylint: disable=no-name-in-module - ResponseCreatedEvent, - ResponseErrorEvent, - ResponseFailedEvent, - ResponseIncompleteEvent, - ResponseInProgressEvent, - ) - - _RESPONSE_EVENTS_WITH_RESPONSE = ( - ResponseCreatedEvent, - ResponseInProgressEvent, - ResponseFailedEvent, - ResponseIncompleteEvent, - ResponseCompletedEvent, - ) -except ImportError: - ResponseCompletedEvent = None - ResponseCreatedEvent = None - ResponseErrorEvent = None - ResponseFailedEvent = None - ResponseIncompleteEvent = None - ResponseInProgressEvent = None - _RESPONSE_EVENTS_WITH_RESPONSE = () +from opentelemetry.util.genai.types import Error try: from opentelemetry.instrumentation.openai_v2.response_extractors import ( # pylint: disable=no-name-in-module @@ -72,7 +37,7 @@ def _set_response_attributes( - invocation: "LLMInvocation", + invocation, result: "ParsedResponse[TextFormatT] | Response | None", capture_content: bool, ) -> None: @@ -131,12 +96,10 @@ class ResponseStreamWrapper(Generic[TextFormatT]): def __init__( self, stream: "ResponseStream[TextFormatT]", - handler: TelemetryHandler, - invocation: "LLMInvocation", + invocation, capture_content: bool, ): self.stream = stream - self.handler = handler self.invocation = invocation self._capture_content = capture_content self._finalized = False @@ -191,9 +154,8 @@ def until_done(self) -> "ResponseStreamWrapper": return self def parse(self) -> "ResponseStreamWrapper": - raise NotImplementedError( - "ResponseStreamWrapper.parse() is not implemented" - ) + """Called when using with_raw_response with stream=True.""" + return self # TODO: Replace __getattr__ passthrough with wrapt.ObjectProxy in a future # cleanup once wrapt 2 typing support is available (wrapt PR #3903). @@ -216,17 +178,15 @@ def _stop( _set_response_attributes( self.invocation, result, self._capture_content ) - with self._safe_instrumentation("stop_llm"): - self.handler.stop_llm(self.invocation) + with self._safe_instrumentation("inference.stop"): + self.invocation.stop() self._finalized = True def _fail(self, message: str, error_type: type[BaseException]) -> None: if self._finalized: return - with self._safe_instrumentation("fail_llm"): - self.handler.fail_llm( - self.invocation, Error(message=message, type=error_type) - ) + with self._safe_instrumentation("inference.fail"): + self.invocation.fail(Error(message=message, type=error_type)) self._finalized = True @staticmethod @@ -244,21 +204,20 @@ def _safe_instrumentation(context: str) -> Generator[None, None, None]: def process_event(self, event: "ResponseStreamEvent[TextFormatT]") -> None: event_type = event.type - response: "ParsedResponse[TextFormatT] | Response | None" = None - - if isinstance(event, _RESPONSE_EVENTS_WITH_RESPONSE): - response = event.response + response: "ParsedResponse[TextFormatT] | Response | None" = getattr( + event, "response", None + ) if response and not self.invocation.request_model: model = response.model if model: self.invocation.request_model = model - if isinstance(event, ResponseCompletedEvent): + if event_type == "response.completed": self._stop(response) return - if isinstance(event, (ResponseFailedEvent, ResponseIncompleteEvent)): + if event_type in {"response.failed", "response.incomplete"}: with self._safe_instrumentation("response attribute extraction"): _set_response_attributes( self.invocation, response, self._capture_content @@ -266,9 +225,9 @@ def process_event(self, event: "ResponseStreamEvent[TextFormatT]") -> None: self._fail(event_type, RuntimeError) return - if isinstance(event, ResponseErrorEvent): - error_type = event.code or "response.error" - message = event.message or error_type + if event_type == "response.error": + error_type = getattr(event, "code", None) or "response.error" + message = getattr(event, "message", None) or error_type self._fail(message, RuntimeError) @@ -282,12 +241,10 @@ class ResponseStreamManagerWrapper(Generic[TextFormatT]): def __init__( self, manager: "ResponseStreamManager[TextFormatT]", - handler: TelemetryHandler, - invocation: "LLMInvocation", + invocation, capture_content: bool, ): self._manager = manager - self._handler = handler self._invocation = invocation self._capture_content = capture_content self._stream_wrapper: ResponseStreamWrapper[TextFormatT] | None = None @@ -296,7 +253,6 @@ def __enter__(self) -> ResponseStreamWrapper[TextFormatT]: stream = self._manager.__enter__() self._stream_wrapper = ResponseStreamWrapper( stream, - self._handler, self._invocation, self._capture_content, ) @@ -390,9 +346,8 @@ async def until_done(self) -> "AsyncResponseStreamWrapper[TextFormatT]": return self def parse(self) -> "AsyncResponseStreamWrapper[TextFormatT]": - raise NotImplementedError( - "AsyncResponseStreamWrapper.parse() is not implemented" - ) + """Called when using with_raw_response with stream=True.""" + return self @property def response(self): @@ -408,12 +363,10 @@ class AsyncResponseStreamManagerWrapper(Generic[TextFormatT]): def __init__( self, manager: "AsyncResponseStreamManager[TextFormatT]", - handler: TelemetryHandler, - invocation: "LLMInvocation", + invocation, capture_content: bool, ): self._manager = manager - self._handler = handler self._invocation = invocation self._capture_content = capture_content self._stream_wrapper: ( @@ -424,7 +377,6 @@ async def __aenter__(self) -> AsyncResponseStreamWrapper[TextFormatT]: stream = await self._manager.__aenter__() self._stream_wrapper = AsyncResponseStreamWrapper( stream, - self._handler, self._invocation, self._capture_content, ) diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_aggregates_cache_tokens[content_mode0].yaml b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_aggregates_cache_tokens[content_mode0].yaml new file mode 100644 index 0000000000..5f683f2566 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_aggregates_cache_tokens[content_mode0].yaml @@ -0,0 +1,172 @@ +interactions: +- request: + body: |- + { + "input": "Say this is a test", + "instructions": "You are a helpful assistant.", + "model": "gpt-4o-mini" + } + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '103' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: |- + { + "id": "resp_0a8171a24e10956f0069e2f3f03a4c819da6eca7c5d387a3b8", + "object": "response", + "created_at": 1776481264, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481265, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0a8171a24e10956f0069e2f3f0f640819d9b572b43c703a798", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06c3d2ae3f2f9-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:01:05 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '1575' + openai-organization: test_openai_org_id + openai-processing-ms: + - '917' + openai-project: + - proj_s74VWObPgWXRchv2sHdrOTPY + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=nawHaNe2Gcjnf3PnJWOe7eCMcuL4GfmHJjNVPemFYOE-1776481264.1903315-1.0.1.1-cmimAw8su.1UpRhI8dWFr6DhgPw2Z2sJi8cKNV_tHFMFijmHK3ND45DjgKsf_ur_EZexZIAN.O__1bqzwnC_acPUsfxzivCaUIWeL1ThD2hWwU2mRBTfDdUDbR04KBsW; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Sat, 18 Apr 2026 + 03:31:05 GMT + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9996' + x-ratelimit-remaining-tokens: + - '199959' + x-ratelimit-reset-requests: + - 34.066s + x-ratelimit-reset-tokens: + - 12ms + x-request-id: + - req_07ac89c9040840a6be685694eacd3288 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_api_error[content_mode0].yaml b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_api_error[content_mode0].yaml new file mode 100644 index 0000000000..2cf338a501 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_api_error[content_mode0].yaml @@ -0,0 +1,94 @@ +interactions: +- request: + body: |- + { + "input": "Hello", + "model": "this-model-does-not-exist" + } + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '56' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: |- + { + "error": { + "message": "The requested model 'this-model-does-not-exist' does not exist.", + "type": "invalid_request_error", + "param": "model", + "code": "model_not_found" + } + } + headers: + CF-RAY: + - 9ee06c562f5c0f7d-EWR + Connection: + - keep-alive + Content-Length: + - '191' + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:01:08 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '107' + openai-project: + - proj_s74VWObPgWXRchv2sHdrOTPY + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=sImUBrv0bwxemWtgUZK8rUGOArohh1idXm1s6ve.9PM-1776481268.1848395-1.0.1.1-wfALia92HJtSuU5STTxRZGW0Cd1tZtPVOpbKTSFoiJLjLOcwVrsTA4C4j2yaYyhAqXvYCG9Ns.EIIhy5ZQv_PNuB11_p2HjTGRNJGtclgs_deByUIWyOSuE2JuY9zxD.; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Sat, 18 Apr 2026 + 03:31:08 GMT + x-request-id: + - req_48f6dc739d654972b34e069edb97d2dc + status: + code: 400 + message: Bad Request +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_basic[content_mode0].yaml b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_basic[content_mode0].yaml new file mode 100644 index 0000000000..c821fcee85 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_basic[content_mode0].yaml @@ -0,0 +1,173 @@ +interactions: +- request: + body: |- + { + "input": "Say this is a test", + "instructions": "You are a helpful assistant.", + "model": "gpt-4o-mini", + "stream": false + } + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '120' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e7b2b88190bff23981628ac362", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '1575' + openai-organization: test_openai_org_id + openai-processing-ms: + - '7177' + openai-project: + - proj_s74VWObPgWXRchv2sHdrOTPY + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=ckwOJyTtv0EpPspvwSIW3P1dXrhIU1GtdL9BNBJlNbA-1776481250.670554-1.0.1.1-7j2fYPA6kw.SMgF3tYMODcA6O.P9Xwu6N6jolKR9OG0AJczT7a1Jr5HlRnJ6T.tXJ5HYV_jtosjzUCBM1yHZTiRTwGTtu6Klyq1ldSG5hFV4MRK9StK_zmCKjmIbqO9d; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Sat, 18 Apr 2026 + 03:30:58 GMT + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '199958' + x-ratelimit-reset-requests: + - 8.64s + x-ratelimit-reset-tokens: + - 12ms + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_captures_content[content_mode0].yaml b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_captures_content[content_mode0].yaml new file mode 100644 index 0000000000..c1d95130d0 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_captures_content[content_mode0].yaml @@ -0,0 +1,178 @@ +interactions: +- request: + body: |- + { + "input": "Say this is a test", + "instructions": "You are a helpful assistant.", + "model": "gpt-4o-mini", + "stream": false, + "text": { + "format": { + "type": "text" + } + } + } + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '158' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: |- + { + "id": "resp_0d6d73520bd9405c0069e2f3eb1108819590a9e98dba6a746d", + "object": "response", + "created_at": 1776481259, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481260, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0d6d73520bd9405c0069e2f3eba7f081958368e178f0078326", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test. How can I assist you further?" + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 13, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 35 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06c1ccc4b0f93-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:01:00 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '1606' + openai-organization: test_openai_org_id + openai-processing-ms: + - '1097' + openai-project: + - proj_s74VWObPgWXRchv2sHdrOTPY + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=eRjb5xrZMpFJHlBLeAzQEHihWjr.Mb5bNSgOGzGSnwA-1776481259.0045145-1.0.1.1-fAERlGPpEWHNuG0MiIcvmzII8t4Xj7yC_HgjLktqdKXPJmbHLtPtT_beeOd8sCcZVQYjGOH0QcF0w4Hz9S.t29trpZuNiezPTQNuyMSy51F2kKYkypy9Eromd2kQp9Cj; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Sat, 18 Apr 2026 + 03:31:00 GMT + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9998' + x-ratelimit-remaining-tokens: + - '199958' + x-ratelimit-reset-requests: + - 13.065s + x-ratelimit-reset-tokens: + - 12ms + x-request-id: + - req_9191bbf984a54a3281f8f335b87e7aeb + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_captures_tool_call_content[content_mode0].yaml b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_captures_tool_call_content[content_mode0].yaml new file mode 100644 index 0000000000..3978b0a80c --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_captures_tool_call_content[content_mode0].yaml @@ -0,0 +1,213 @@ +interactions: +- request: + body: |- + { + "input": "What's the weather in Seattle right now?", + "model": "gpt-4o-mini", + "tool_choice": { + "type": "function", + "name": "get_current_weather" + }, + "tools": [ + { + "type": "function", + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. Boston, MA" + } + }, + "required": [ + "location" + ], + "additionalProperties": false + }, + "strict": true + } + ] + } + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '480' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: |- + { + "id": "resp_0bedf6e1ffba28050069e2f401ae1c8196be360fd5993c96de", + "object": "response", + "created_at": 1776481281, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481282, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "fc_0bedf6e1ffba28050069e2f402203c8196b45065342d00ed17", + "type": "function_call", + "status": "completed", + "arguments": "{\"location\":\"Seattle, WA\"}", + "call_id": "call_90uO5LcGP5vTBTCrjyhYtWsA", + "name": "get_current_weather" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": { + "type": "function", + "name": "get_current_weather" + }, + "tools": [ + { + "type": "function", + "description": "Get the current weather in a given location", + "name": "get_current_weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. Boston, MA" + } + }, + "required": [ + "location" + ], + "additionalProperties": false + }, + "strict": true + } + ], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 72, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 80 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06caa3d06c745-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:01:22 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '2034' + openai-organization: test_openai_org_id + openai-processing-ms: + - '713' + openai-project: + - proj_s74VWObPgWXRchv2sHdrOTPY + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=77TblxQ40FwSWcmbiooIxTpBbJWtYa4Sa8xnnuOb8P4-1776481281.6342883-1.0.1.1-wYyCi1iDQ3bB4g5c2Jj1pldvzLmLS9NtJ1ryP99rJkxwR77VRG3qXvTHSKzUAQk9LBvwdBumDx4dWSKvmBqXsb7K2O8zcH3uxfGFCw4SLKIaQIsDy2SGcb9JuoVZMe4r; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Sat, 18 Apr 2026 + 03:31:22 GMT + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9987' + x-ratelimit-remaining-tokens: + - '199711' + x-ratelimit-reset-requests: + - 1m51.595s + x-ratelimit-reset-tokens: + - 86ms + x-request-id: + - req_66f20a8d196540e7928ba09ef1baad00 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_event_only_no_content_in_span.yaml b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_event_only_no_content_in_span.yaml new file mode 100644 index 0000000000..729f09d913 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_event_only_no_content_in_span.yaml @@ -0,0 +1,173 @@ +interactions: +- request: + body: |- + { + "input": "Say this is a test", + "instructions": "You are a helpful assistant.", + "model": "gpt-4o-mini", + "stream": false + } + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '120' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: |- + { + "id": "resp_0da562f2b64fdb7e0069e2f40b28e081909487104de4d4c8c0", + "object": "response", + "created_at": 1776481291, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481291, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0da562f2b64fdb7e0069e2f40ba8bc8190a803f1b565d49c7c", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06ce2aead41b2-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:01:31 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '1575' + openai-organization: test_openai_org_id + openai-processing-ms: + - '698' + openai-project: + - proj_s74VWObPgWXRchv2sHdrOTPY + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=5Y3wm6qm03P0G7_SigGuGob8rfE0X7eMfHOJDLR_E5o-1776481290.6632602-1.0.1.1-2QRz.JRmnCza_Xtkyqwgj1IPTwrSSA5LHyD9chhvKiCoHNE9_.I5_0qopgAV60mriRXdX99CCxozoio7VqvW.wnvklHzb6gwZDEsQCAlBqZGcIHrBfFk4jkvspj4sMO0; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Sat, 18 Apr 2026 + 03:31:31 GMT + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9985' + x-ratelimit-remaining-tokens: + - '199959' + x-ratelimit-reset-requests: + - 2m7.93s + x-ratelimit-reset-tokens: + - 12ms + x-request-id: + - req_76d5ef520a6b4d74975dae1ade822de9 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_instrumentation_error_swallowed[content_mode0].yaml b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_instrumentation_error_swallowed[content_mode0].yaml new file mode 100644 index 0000000000..12dd8a9564 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_instrumentation_error_swallowed[content_mode0].yaml @@ -0,0 +1,127 @@ +interactions: +- request: + body: |- + { + "input": "Say this is a test", + "instructions": "You are a helpful assistant.", + "model": "gpt-4o-mini", + "stream": true + } + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '119' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: |+ + event: response.created + data: {"type":"response.created","response":{"id":"resp_052003e5c7cd2fe10069e2f400498c819f943227bcc94a63a6","object":"response","created_at":1776481280,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"You are a helpful assistant.","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + event: response.in_progress + data: {"type":"response.in_progress","response":{"id":"resp_052003e5c7cd2fe10069e2f400498c819f943227bcc94a63a6","object":"response","created_at":1776481280,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"You are a helpful assistant.","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + event: response.output_item.added + data: {"type":"response.output_item.added","item":{"id":"msg_052003e5c7cd2fe10069e2f400ec70819f90c4df2a4b73f05c","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":0,"sequence_number":2} + + event: response.content_part.added + data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_052003e5c7cd2fe10069e2f400ec70819f90c4df2a4b73f05c","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","content_index":0,"delta":"This","item_id":"msg_052003e5c7cd2fe10069e2f400ec70819f90c4df2a4b73f05c","logprobs":[],"obfuscation":"hmEwhe4L8nm4","output_index":0,"sequence_number":4} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","content_index":0,"delta":" is","item_id":"msg_052003e5c7cd2fe10069e2f400ec70819f90c4df2a4b73f05c","logprobs":[],"obfuscation":"u3Q5kHUIahzcI","output_index":0,"sequence_number":5} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","content_index":0,"delta":" a","item_id":"msg_052003e5c7cd2fe10069e2f400ec70819f90c4df2a4b73f05c","logprobs":[],"obfuscation":"0ZnBkOCdgrlbIT","output_index":0,"sequence_number":6} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","content_index":0,"delta":" test","item_id":"msg_052003e5c7cd2fe10069e2f400ec70819f90c4df2a4b73f05c","logprobs":[],"obfuscation":"hYsQ5M0hWLs","output_index":0,"sequence_number":7} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","content_index":0,"delta":".","item_id":"msg_052003e5c7cd2fe10069e2f400ec70819f90c4df2a4b73f05c","logprobs":[],"obfuscation":"PyM68hgEwGGLkTN","output_index":0,"sequence_number":8} + + event: response.output_text.done + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_052003e5c7cd2fe10069e2f400ec70819f90c4df2a4b73f05c","logprobs":[],"output_index":0,"sequence_number":9,"text":"This is a test."} + + event: response.content_part.done + data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_052003e5c7cd2fe10069e2f400ec70819f90c4df2a4b73f05c","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"This is a test."},"sequence_number":10} + + event: response.output_item.done + data: {"type":"response.output_item.done","item":{"id":"msg_052003e5c7cd2fe10069e2f400ec70819f90c4df2a4b73f05c","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"This is a test."}],"role":"assistant"},"output_index":0,"sequence_number":11} + + event: response.completed + data: {"type":"response.completed","response":{"id":"resp_052003e5c7cd2fe10069e2f400498c819f943227bcc94a63a6","object":"response","created_at":1776481280,"status":"completed","background":false,"completed_at":1776481280,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"You are a helpful assistant.","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[{"id":"msg_052003e5c7cd2fe10069e2f400ec70819f90c4df2a4b73f05c","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"This is a test."}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":22,"input_tokens_details":{"cached_tokens":0},"output_tokens":6,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":28},"user":null,"metadata":{}},"sequence_number":12} + + headers: + CF-RAY: + - 9ee06ca19ad7606a-EWR + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Sat, 18 Apr 2026 03:01:20 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '51' + openai-project: + - proj_s74VWObPgWXRchv2sHdrOTPY + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=zeme85udcp85MpYWX94kpho5aA4pRXRhhWbljEzlyKg-1776481280.2544615-1.0.1.1-DxHmuQ9bjw1.C_t_MZ23Az8WRay9CC2Wh0GbHPNN41zA5npmFYATDR2lI7uN061efp8b0vaq0viZms.aHHjkxiiNYFhewlDgEVLyFCa4F1mVx1iKbXQcmUJWbaXjFOtV; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Sat, 18 Apr 2026 + 03:31:20 GMT + x-request-id: + - req_4e7549325e3f4abaa4ac13b74667292e + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_reports_reasoning_tokens[content_mode0].yaml b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_reports_reasoning_tokens[content_mode0].yaml new file mode 100644 index 0000000000..11739e2d3e --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_responses_create_reports_reasoning_tokens[content_mode0].yaml @@ -0,0 +1,186 @@ +interactions: +- request: + body: |- + { + "input": [ + { + "role": "user", + "content": "\nWrite a bash script that takes a matrix represented as a string with\nformat '[1,2],[3,4],[5,6]' and prints the transpose in the same format.\n" + } + ], + "max_output_tokens": 300, + "model": "gpt-5.4", + "reasoning": { + "effort": "low" + } + } + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '267' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '30.0' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: |- + { + "id": "resp_05177a4994c7df3a0069e2f402f00881a1b9eda520cb779fef", + "object": "response", + "created_at": 1776481282, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481288, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 300, + "max_tool_calls": null, + "model": "gpt-5.4-2026-03-05", + "output": [ + { + "id": "rs_05177a4994c7df3a0069e2f403351c81a1ae31439661e8e976", + "type": "reasoning", + "summary": [] + }, + { + "id": "msg_05177a4994c7df3a0069e2f403633481a1a824dd7ecef75116", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "```bash\n#!/usr/bin/env bash\n\n# Usage:\n# ./transpose.sh '[1,2],[3,4],[5,6]'\n\ninput=\"$1\"\n\n# Extract rows into an array\nIFS='|' read -r -a rows <<< \"$(echo \"$input\" | sed 's/],\\[/]|[/g')\"\n\n# Remove leading/trailing brackets and split first row to get column count\nfirst_row=\"${rows[0]#[}\"\nfirst_row=\"${first_row%]}\"\nIFS=',' read -r -a first_vals <<< \"$first_row\"\ncols=\"${#first_vals[@]}\"\n\n# Build transpose\nresult=\"\"\nfor ((c=0; c 0 + + (span,) = span_exporter.get_finished_spans() + assert ( + span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == DEFAULT_MODEL + ) + assert GenAIAttributes.GEN_AI_RESPONSE_ID in span.attributes + assert GenAIAttributes.GEN_AI_RESPONSE_MODEL in span.attributes + + +@pytest.mark.vcr() +def test_responses_create_streaming_delegates_response_attribute( + request, openai_client, instrument_no_content +): + _skip_if_not_latest() + + stream = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input="Say hi.", + stream=True, + ) + + assert stream.response is not None + assert stream.response.status_code == 200 + assert stream.response.headers.get("x-request-id") is not None + stream.close() + + +def test_responses_create_streaming_connection_error( + span_exporter, instrument_no_content +): + _skip_if_not_latest() + + client = OpenAI(base_url="http://localhost:4242") + + with pytest.raises(APIConnectionError): + client.responses.create( # pylint: disable=no-member + model=DEFAULT_MODEL, + input="Hello", + stream=True, + timeout=0.1, + ) + + (span,) = span_exporter.get_finished_spans() + assert ( + span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == DEFAULT_MODEL + ) + assert span.attributes[ErrorAttributes.ERROR_TYPE] == "APIConnectionError" + + +@pytest.mark.vcr() +def test_responses_stream_wrapper_finalize_idempotent( + request, span_exporter, openai_client, instrument_no_content +): + _skip_if_not_latest() + + stream = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=True, + ) + + response = _collect_completed_response(stream) + stream.close() + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + assert_all_attributes( + spans[0], + DEFAULT_MODEL, + True, + response.id, + response.model, + response.usage.input_tokens, + response.usage.output_tokens, + response_service_tier=getattr(response, "service_tier", None), + ) + + +@pytest.mark.vcr() +def test_responses_create_stream_propagation_error( + request, span_exporter, openai_client, instrument_no_content, monkeypatch +): + _skip_if_not_latest() + + stream = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=True, + ) + + class ErrorInjectingStreamDelegate: + def __init__(self, inner): + self._inner = inner + self._count = 0 + + def __iter__(self): + return self + + def __next__(self): + if self._count == 1: + raise ConnectionError("connection reset during stream") + self._count += 1 + return next(self._inner) + + def close(self): + return self._inner.close() + + def __getattr__(self, name): + return getattr(self._inner, name) + + monkeypatch.setattr( + stream, "stream", ErrorInjectingStreamDelegate(stream.stream) + ) + + with pytest.raises( + ConnectionError, match="connection reset during stream" + ): + with stream: + for _ in stream: + pass + + (span,) = span_exporter.get_finished_spans() + assert ( + span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == DEFAULT_MODEL + ) + assert span.attributes[ErrorAttributes.ERROR_TYPE] == "ConnectionError" + + +@pytest.mark.vcr() +def test_responses_create_streaming_user_exception( + request, span_exporter, openai_client, instrument_no_content +): + _skip_if_not_latest() + + with pytest.raises(ValueError, match="User raised exception"): + with openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=True, + ) as stream: + for _ in stream: + raise ValueError("User raised exception") + + (span,) = span_exporter.get_finished_spans() + assert ( + span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == DEFAULT_MODEL + ) + assert span.attributes[ErrorAttributes.ERROR_TYPE] == "ValueError" + + +@pytest.mark.vcr() +def test_responses_create_instrumentation_error_swallowed( + request, span_exporter, openai_client, instrument_no_content, monkeypatch +): + _skip_if_not_latest() + + def exploding_process_event(self, event): + del self + del event + raise RuntimeError("instrumentation bug") + + monkeypatch.setattr( + ResponseStreamWrapper, "process_event", exploding_process_event + ) + + with openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=True, + ) as stream: + events = list(stream) + + assert len(events) > 0 + + (span,) = span_exporter.get_finished_spans() + assert ( + span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == DEFAULT_MODEL + ) + assert ErrorAttributes.ERROR_TYPE not in span.attributes + + +@pytest.mark.vcr() +@pytest.mark.skipif( + not _has_tools_param, + reason="openai SDK too old to support 'tools' parameter on Responses.create", +) +def test_responses_create_captures_tool_call_content( + request, span_exporter, openai_client, instrument_with_content +): + _skip_if_not_latest() + + openai_client.responses.create( + model=DEFAULT_MODEL, + input="What's the weather in Seattle right now?", + tools=[get_responses_weather_tool_definition()], + tool_choice={"type": "function", "name": "get_current_weather"}, + ) + + (span,) = span_exporter.get_finished_spans() + output_messages = _load_span_messages( + span, GenAIAttributes.GEN_AI_OUTPUT_MESSAGES + ) + assert any( + part.get("type") == "tool_call" + for message in output_messages + for part in message.get("parts", []) + ) + + +@pytest.mark.vcr() +@pytest.mark.skipif( + not _has_reasoning_param, + reason=( + "openai SDK too old to support 'reasoning' parameter on Responses.create" + ), +) +def test_responses_create_reports_reasoning_tokens( + request, span_exporter, openai_client, instrument_with_content +): + _skip_if_not_latest() + + response = openai_client.responses.create( + model=REASONING_MODEL, + reasoning={"effort": "low"}, + input=[ + { + "role": "user", + "content": REASONING_PROMPT, + } + ], + max_output_tokens=300, + timeout=30.0, + ) + + reasoning_tokens = getattr( + getattr(response.usage, "output_tokens_details", None), + "reasoning_tokens", + None, + ) + + assert reasoning_tokens is not None + assert reasoning_tokens > 0 + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + (span,) = spans + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == ( + REASONING_MODEL + ) + + +@pytest.mark.vcr() +def test_responses_create_with_content_span_unsampled( + request, + span_exporter, + log_exporter, + openai_client, + instrument_with_content_unsampled, +): + _skip_if_not_latest() + + openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=False, + ) + + assert len(span_exporter.get_finished_spans()) == 0 + assert len(log_exporter.get_finished_logs()) == 0 + + +@pytest.mark.vcr() +def test_responses_create_with_content_shapes( + request, + span_exporter, + log_exporter, + openai_client, + instrument_with_content, +): + _skip_if_not_latest() + + openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=False, + ) + + (span,) = span_exporter.get_finished_spans() + input_messages = _load_span_messages( + span, GenAIAttributes.GEN_AI_INPUT_MESSAGES + ) + output_messages = _load_span_messages( + span, GenAIAttributes.GEN_AI_OUTPUT_MESSAGES + ) + + assert input_messages[0]["role"] == "user" + assert input_messages[0]["parts"][0]["type"] == "text" + assert output_messages[0]["role"] == "assistant" + assert output_messages[0]["parts"][0]["type"] == "text" + assert len(log_exporter.get_finished_logs()) == 0 + + +@pytest.mark.vcr() +def test_responses_create_event_only_no_content_in_span( + request, span_exporter, log_exporter, openai_client, instrument_event_only +): + _skip_if_not_latest() + + openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=False, + ) + + (span,) = span_exporter.get_finished_spans() + assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes + assert GenAIAttributes.GEN_AI_OUTPUT_MESSAGES not in span.attributes + assert GenAIAttributes.GEN_AI_SYSTEM_INSTRUCTIONS not in span.attributes + + logs = log_exporter.get_finished_logs() + assert len(logs) == 1 + assert ( + logs[0].log_record.event_name + == "gen_ai.client.inference.operation.details" + ) diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_utils.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_utils.py index 211fa3739d..cbb2a538c3 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_utils.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_utils.py @@ -17,6 +17,10 @@ import json from typing import Any, Optional +from opentelemetry.instrumentation.openai_v2.response_extractors import ( + GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, +) from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, @@ -184,6 +188,26 @@ def get_current_weather_tool_definition(): } +def get_responses_weather_tool_definition(): + return { + "type": "function", + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. Boston, MA", + }, + }, + "required": ["location"], + "additionalProperties": False, + }, + "strict": True, + } + + def remove_none_values(body): """Remove None values from a dictionary recursively""" result = {} @@ -244,6 +268,35 @@ def format_simple_expected_output_message( ] +def _get_usage_details(usage): + return getattr(usage, "input_tokens_details", None) or getattr( + usage, "prompt_tokens_details", None + ) + + +def assert_cache_attributes(span, usage): + details = _get_usage_details(usage) + assert details is not None + + cached_tokens = getattr(details, "cached_tokens", None) + if cached_tokens is None: + assert GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS not in span.attributes + else: + assert ( + span.attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] + == cached_tokens + ) + + cache_creation = getattr(details, "cache_creation_input_tokens", None) + if cache_creation is None: + assert GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS not in span.attributes + else: + assert ( + span.attributes[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] + == cache_creation + ) + + def assert_message_in_logs(log, event_name, expected_content, parent_span): assert log.log_record.event_name == event_name assert (