diff --git a/CHANGELOG.md b/CHANGELOG.md index 93f0cab4fc..ffac555530 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `opentelemetry-instrumentation-openai-v2`: Add instrumentation for `chat.completions.parse()` structured outputs + ([#4416](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4416)) - Bump `pylint` to `4.0.5` ([#4244](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4244)) - `opentelemetry-instrumentation-sqlite3`: Add uninstrument, error status, suppress, and no-op tests diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py index afebf95cb2..0308999c40 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py @@ -104,9 +104,25 @@ ) +def _is_parse_supported(): + """Check if the parse() method is available on the Completions class. + + The parse() method for structured outputs was added in openai >= 1.40.0. + """ + try: + from openai.resources.chat.completions import ( # pylint: disable=import-outside-toplevel # noqa: PLC0415 + Completions, + ) + + return hasattr(Completions, "parse") + except ImportError: + return False + + class OpenAIInstrumentor(BaseInstrumentor): def __init__(self): self._meter = None + self._parse_supported = False def instrumentation_dependencies(self) -> Collection[str]: return _instruments @@ -188,6 +204,36 @@ def _instrument(self, **kwargs): ), ) + # parse() wraps create() internally in the OpenAI SDK and returns a + # ParsedChatCompletion. The telemetry-relevant fields (model, usage, + # choices, finish_reason) are identical to ChatCompletion, so the + # existing create() wrappers handle it correctly. + self._parse_supported = _is_parse_supported() + if self._parse_supported: + wrap_function_wrapper( + "openai.resources.chat.completions", + "Completions.parse", + ( + chat_completions_create_v_new(handler) + if latest_experimental_enabled + else chat_completions_create_v_old( + tracer, logger, instruments, is_content_enabled() + ) + ), + ) + + wrap_function_wrapper( + "openai.resources.chat.completions", + "AsyncCompletions.parse", + ( + async_chat_completions_create_v_new(handler) + if latest_experimental_enabled + else async_chat_completions_create_v_old( + tracer, logger, instruments, is_content_enabled() + ) + ), + ) + def _uninstrument(self, **kwargs): import openai # pylint: disable=import-outside-toplevel # noqa: PLC0415 @@ -195,3 +241,7 @@ def _uninstrument(self, **kwargs): unwrap(openai.resources.chat.completions.AsyncCompletions, "create") unwrap(openai.resources.embeddings.Embeddings, "create") unwrap(openai.resources.embeddings.AsyncEmbeddings, "create") + + if self._parse_supported: + unwrap(openai.resources.chat.completions.Completions, "parse") + unwrap(openai.resources.chat.completions.AsyncCompletions, "parse") diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py index 4dab04d977..b27726faf5 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py @@ -289,8 +289,18 @@ def get_llm_request_attributes( else GenAIAttributes.GEN_AI_OPENAI_REQUEST_RESPONSE_FORMAT ) if (response_format := kwargs.get("response_format")) is not None: - # response_format may be string or object with a string in the `type` key - if isinstance(response_format, Mapping): + # response_format may be string, object with a string in the `type` key, + # or a type (e.g. Pydantic model class used with parse()) + if isinstance(response_format, type): + if latest_experimental_enabled: + attributes[request_response_format_attr_key] = ( + GenAIAttributes.GenAiOutputTypeValues.JSON.value + ) + else: + attributes[request_response_format_attr_key] = ( + GenAIAttributes.GenAiOpenaiRequestResponseFormatValues.JSON_SCHEMA.value + ) + elif isinstance(response_format, Mapping): if ( response_format_type := response_format.get("type") ) is not None: @@ -378,8 +388,13 @@ def create_chat_invocation( if ( response_format := get_value(kwargs.get("response_format")) ) is not None: - # response_format may be string or object with a string in the `type` key - if isinstance(response_format, Mapping): + # response_format may be string, object with a string in the `type` key, + # or a type (e.g. Pydantic model class used with parse()) + if isinstance(response_format, type): + invocation.attributes[GenAIAttributes.GEN_AI_OUTPUT_TYPE] = ( + GenAIAttributes.GenAiOutputTypeValues.JSON.value + ) + elif isinstance(response_format, Mapping): if ( response_format_type := get_value(response_format.get("type")) ) is not None: diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_async_structured_output_no_content.yaml b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_async_structured_output_no_content.yaml new file mode 100644 index 0000000000..bd888126ac --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_async_structured_output_no_content.yaml @@ -0,0 +1,116 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "user", + "content": "Extract the event information from: Team Meeting on 2024-01-15 with Alice and Bob" + } + ], + "model": "gpt-4o-mini", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "CalendarEvent", + "strict": true, + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "date": {"type": "string"}, + "participants": {"items": {"type": "string"}, "type": "array"} + }, + "required": ["name", "date", "participants"], + "additionalProperties": false + } + } + } + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.54.3 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "id": "chatcmpl-structured-test-004", + "object": "chat.completion", + "created": 1731368630, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "{\"name\": \"Team Meeting\", \"date\": \"2024-01-15\", \"participants\": [\"Alice\", \"Bob\"]}", + "refusal": null + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 50, + "completion_tokens": 30, + "total_tokens": 80, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "system_fingerprint": "fp_0ba0d124f1" + } + headers: + CF-Cache-Status: + - DYNAMIC + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Mon, 11 Nov 2024 23:43:50 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + content-length: + - '800' + openai-organization: test_openai_org_id + openai-processing-ms: + - '350' + openai-version: + - '2020-10-01' + x-request-id: + - req_structured_test_004 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_async_structured_output_with_content.yaml b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_async_structured_output_with_content.yaml new file mode 100644 index 0000000000..0e01efec3a --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_async_structured_output_with_content.yaml @@ -0,0 +1,116 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "user", + "content": "Extract the event information from: Team Meeting on 2024-01-15 with Alice and Bob" + } + ], + "model": "gpt-4o-mini", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "CalendarEvent", + "strict": true, + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "date": {"type": "string"}, + "participants": {"items": {"type": "string"}, "type": "array"} + }, + "required": ["name", "date", "participants"], + "additionalProperties": false + } + } + } + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.54.3 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "id": "chatcmpl-structured-test-003", + "object": "chat.completion", + "created": 1731368630, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "{\"name\": \"Team Meeting\", \"date\": \"2024-01-15\", \"participants\": [\"Alice\", \"Bob\"]}", + "refusal": null + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 50, + "completion_tokens": 30, + "total_tokens": 80, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "system_fingerprint": "fp_0ba0d124f1" + } + headers: + CF-Cache-Status: + - DYNAMIC + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Mon, 11 Nov 2024 23:43:50 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + content-length: + - '800' + openai-organization: test_openai_org_id + openai-processing-ms: + - '350' + openai-version: + - '2020-10-01' + x-request-id: + - req_structured_test_003 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_structured_output_no_content.yaml b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_structured_output_no_content.yaml new file mode 100644 index 0000000000..694877e779 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_structured_output_no_content.yaml @@ -0,0 +1,116 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "user", + "content": "Extract the event information from: Team Meeting on 2024-01-15 with Alice and Bob" + } + ], + "model": "gpt-4o-mini", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "CalendarEvent", + "strict": true, + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "date": {"type": "string"}, + "participants": {"items": {"type": "string"}, "type": "array"} + }, + "required": ["name", "date", "participants"], + "additionalProperties": false + } + } + } + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.54.3 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "id": "chatcmpl-structured-test-002", + "object": "chat.completion", + "created": 1731368630, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "{\"name\": \"Team Meeting\", \"date\": \"2024-01-15\", \"participants\": [\"Alice\", \"Bob\"]}", + "refusal": null + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 50, + "completion_tokens": 30, + "total_tokens": 80, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "system_fingerprint": "fp_0ba0d124f1" + } + headers: + CF-Cache-Status: + - DYNAMIC + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Mon, 11 Nov 2024 23:43:50 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + content-length: + - '800' + openai-organization: test_openai_org_id + openai-processing-ms: + - '350' + openai-version: + - '2020-10-01' + x-request-id: + - req_structured_test_002 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_structured_output_with_content.yaml b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_structured_output_with_content.yaml new file mode 100644 index 0000000000..066a17e139 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_structured_output_with_content.yaml @@ -0,0 +1,116 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "user", + "content": "Extract the event information from: Team Meeting on 2024-01-15 with Alice and Bob" + } + ], + "model": "gpt-4o-mini", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "CalendarEvent", + "strict": true, + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "date": {"type": "string"}, + "participants": {"items": {"type": "string"}, "type": "array"} + }, + "required": ["name", "date", "participants"], + "additionalProperties": false + } + } + } + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.54.3 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "id": "chatcmpl-structured-test-001", + "object": "chat.completion", + "created": 1731368630, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "{\"name\": \"Team Meeting\", \"date\": \"2024-01-15\", \"participants\": [\"Alice\", \"Bob\"]}", + "refusal": null + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 50, + "completion_tokens": 30, + "total_tokens": 80, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "system_fingerprint": "fp_0ba0d124f1" + } + headers: + CF-Cache-Status: + - DYNAMIC + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Mon, 11 Nov 2024 23:43:50 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + content-length: + - '800' + openai-organization: test_openai_org_id + openai-processing-ms: + - '350' + openai-version: + - '2020-10-01' + x-request-id: + - req_structured_test_001 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/structured_outputs_utils.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/structured_outputs_utils.py new file mode 100644 index 0000000000..c150f947d1 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/structured_outputs_utils.py @@ -0,0 +1,45 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared test definitions for structured outputs (parse) tests.""" + +from pydantic import BaseModel + + +class CalendarEvent(BaseModel): + name: str + date: str + participants: list[str] + + +STRUCTURED_OUTPUT_PROMPT = [ + { + "role": "user", + "content": "Extract the event information from: Team Meeting on 2024-01-15 with Alice and Bob", + } +] + +STRUCTURED_OUTPUT_EXPECTED_INPUT_MESSAGES = [ + { + "role": "user", + "parts": [ + { + "type": "text", + "content": STRUCTURED_OUTPUT_PROMPT[0]["content"], + } + ], + } +] + +EXPECTED_RESPONSE_CONTENT = '{"name": "Team Meeting", "date": "2024-01-15", "participants": ["Alice", "Bob"]}' diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_async_structured_outputs.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_async_structured_outputs.py new file mode 100644 index 0000000000..121953ff1d --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_async_structured_outputs.py @@ -0,0 +1,174 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for async OpenAI structured outputs (chat.completions.parse) instrumentation.""" + +import pytest +from openai.resources.chat.completions import Completions + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) +from opentelemetry.util.genai.utils import is_experimental_mode + +from .structured_outputs_utils import ( + STRUCTURED_OUTPUT_EXPECTED_INPUT_MESSAGES, + STRUCTURED_OUTPUT_PROMPT, + CalendarEvent, +) +from .test_utils import ( + DEFAULT_MODEL, + assert_all_attributes, + assert_message_in_logs, + assert_messages_attribute, + format_simple_expected_output_message, +) + +pytestmark = pytest.mark.skipif( + not hasattr(Completions, "parse"), + reason="parse() requires openai >= 1.40.0", +) + + +@pytest.mark.asyncio() +async def test_async_structured_output_with_content( + span_exporter, + log_exporter, + async_openai_client, + instrument_with_content, + vcr, +): + latest_experimental_enabled = is_experimental_mode() + + with vcr.use_cassette("test_async_structured_output_with_content.yaml"): + response = await async_openai_client.chat.completions.parse( + messages=STRUCTURED_OUTPUT_PROMPT, + model=DEFAULT_MODEL, + response_format=CalendarEvent, + ) + + # Verify wrapper doesn't interfere with parse() return + assert response.choices[0].message.parsed is not None + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + assert_all_attributes( + spans[0], + DEFAULT_MODEL, + latest_experimental_enabled, + response.id, + response.model, + response.usage.prompt_tokens, + response.usage.completion_tokens, + ) + + output_type_attr_key = ( + GenAIAttributes.GEN_AI_OUTPUT_TYPE + if latest_experimental_enabled + else GenAIAttributes.GEN_AI_OPENAI_REQUEST_RESPONSE_FORMAT + ) + expected_value = "json" if latest_experimental_enabled else "json_schema" + assert spans[0].attributes[output_type_attr_key] == expected_value + + if latest_experimental_enabled: + assert_messages_attribute( + spans[0].attributes["gen_ai.input.messages"], + STRUCTURED_OUTPUT_EXPECTED_INPUT_MESSAGES, + ) + assert_messages_attribute( + spans[0].attributes["gen_ai.output.messages"], + format_simple_expected_output_message( + response.choices[0].message.content + ), + ) + else: + logs = log_exporter.get_finished_logs() + assert len(logs) == 2 + + user_message = {"content": STRUCTURED_OUTPUT_PROMPT[0]["content"]} + assert_message_in_logs( + logs[0], "gen_ai.user.message", user_message, spans[0] + ) + + choice_event = { + "index": 0, + "finish_reason": "stop", + "message": { + "role": "assistant", + "content": response.choices[0].message.content, + }, + } + assert_message_in_logs( + logs[1], "gen_ai.choice", choice_event, spans[0] + ) + + +@pytest.mark.asyncio() +async def test_async_structured_output_no_content( + span_exporter, + log_exporter, + async_openai_client, + instrument_no_content, + vcr, +): + latest_experimental_enabled = is_experimental_mode() + + with vcr.use_cassette("test_async_structured_output_no_content.yaml"): + response = await async_openai_client.chat.completions.parse( + messages=STRUCTURED_OUTPUT_PROMPT, + model=DEFAULT_MODEL, + response_format=CalendarEvent, + ) + + # Verify wrapper doesn't interfere with parse() return + assert response.choices[0].message.parsed is not None + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + assert_all_attributes( + spans[0], + DEFAULT_MODEL, + latest_experimental_enabled, + response.id, + response.model, + response.usage.prompt_tokens, + response.usage.completion_tokens, + ) + + output_type_attr_key = ( + GenAIAttributes.GEN_AI_OUTPUT_TYPE + if latest_experimental_enabled + else GenAIAttributes.GEN_AI_OPENAI_REQUEST_RESPONSE_FORMAT + ) + expected_value = "json" if latest_experimental_enabled else "json_schema" + assert spans[0].attributes[output_type_attr_key] == expected_value + + logs = log_exporter.get_finished_logs() + if latest_experimental_enabled: + assert len(logs) == 0 + assert "gen_ai.input.messages" not in spans[0].attributes + assert "gen_ai.output.messages" not in spans[0].attributes + else: + assert len(logs) == 2 + + assert_message_in_logs(logs[0], "gen_ai.user.message", None, spans[0]) + + choice_event = { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant"}, + } + assert_message_in_logs( + logs[1], "gen_ai.choice", choice_event, spans[0] + ) diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_structured_outputs.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_structured_outputs.py new file mode 100644 index 0000000000..173bc7085a --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_structured_outputs.py @@ -0,0 +1,164 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for OpenAI structured outputs (chat.completions.parse) instrumentation.""" + +import pytest +from openai.resources.chat.completions import Completions + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) +from opentelemetry.util.genai.utils import is_experimental_mode + +from .structured_outputs_utils import ( + STRUCTURED_OUTPUT_EXPECTED_INPUT_MESSAGES, + STRUCTURED_OUTPUT_PROMPT, + CalendarEvent, +) +from .test_utils import ( + DEFAULT_MODEL, + assert_all_attributes, + assert_message_in_logs, + assert_messages_attribute, + format_simple_expected_output_message, +) + +pytestmark = pytest.mark.skipif( + not hasattr(Completions, "parse"), + reason="parse() requires openai >= 1.40.0", +) + + +def test_structured_output_with_content( + span_exporter, log_exporter, openai_client, instrument_with_content, vcr +): + latest_experimental_enabled = is_experimental_mode() + + with vcr.use_cassette("test_structured_output_with_content.yaml"): + response = openai_client.chat.completions.parse( + messages=STRUCTURED_OUTPUT_PROMPT, + model=DEFAULT_MODEL, + response_format=CalendarEvent, + ) + + # Verify wrapper doesn't interfere with parse() return + assert response.choices[0].message.parsed is not None + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + assert_all_attributes( + spans[0], + DEFAULT_MODEL, + latest_experimental_enabled, + response.id, + response.model, + response.usage.prompt_tokens, + response.usage.completion_tokens, + ) + + output_type_attr_key = ( + GenAIAttributes.GEN_AI_OUTPUT_TYPE + if latest_experimental_enabled + else GenAIAttributes.GEN_AI_OPENAI_REQUEST_RESPONSE_FORMAT + ) + expected_value = "json" if latest_experimental_enabled else "json_schema" + assert spans[0].attributes[output_type_attr_key] == expected_value + + if latest_experimental_enabled: + assert_messages_attribute( + spans[0].attributes["gen_ai.input.messages"], + STRUCTURED_OUTPUT_EXPECTED_INPUT_MESSAGES, + ) + assert_messages_attribute( + spans[0].attributes["gen_ai.output.messages"], + format_simple_expected_output_message( + response.choices[0].message.content + ), + ) + else: + logs = log_exporter.get_finished_logs() + assert len(logs) == 2 + + user_message = {"content": STRUCTURED_OUTPUT_PROMPT[0]["content"]} + assert_message_in_logs( + logs[0], "gen_ai.user.message", user_message, spans[0] + ) + + choice_event = { + "index": 0, + "finish_reason": "stop", + "message": { + "role": "assistant", + "content": response.choices[0].message.content, + }, + } + assert_message_in_logs( + logs[1], "gen_ai.choice", choice_event, spans[0] + ) + + +def test_structured_output_no_content( + span_exporter, log_exporter, openai_client, instrument_no_content, vcr +): + latest_experimental_enabled = is_experimental_mode() + + with vcr.use_cassette("test_structured_output_no_content.yaml"): + response = openai_client.chat.completions.parse( + messages=STRUCTURED_OUTPUT_PROMPT, + model=DEFAULT_MODEL, + response_format=CalendarEvent, + ) + + # Verify wrapper doesn't interfere with parse() return + assert response.choices[0].message.parsed is not None + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + assert_all_attributes( + spans[0], + DEFAULT_MODEL, + latest_experimental_enabled, + response.id, + response.model, + response.usage.prompt_tokens, + response.usage.completion_tokens, + ) + + output_type_attr_key = ( + GenAIAttributes.GEN_AI_OUTPUT_TYPE + if latest_experimental_enabled + else GenAIAttributes.GEN_AI_OPENAI_REQUEST_RESPONSE_FORMAT + ) + expected_value = "json" if latest_experimental_enabled else "json_schema" + assert spans[0].attributes[output_type_attr_key] == expected_value + + logs = log_exporter.get_finished_logs() + if latest_experimental_enabled: + assert len(logs) == 0 + assert "gen_ai.input.messages" not in spans[0].attributes + assert "gen_ai.output.messages" not in spans[0].attributes + else: + assert len(logs) == 2 + + assert_message_in_logs(logs[0], "gen_ai.user.message", None, spans[0]) + + choice_event = { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant"}, + } + assert_message_in_logs( + logs[1], "gen_ai.choice", choice_event, spans[0] + )