Skip to content

Commit 8e8b16b

Browse files
Nik-ReddyNik-Reddy
committed
Add instrumentation for chat.completions.parse() structured outputs
Wrap Completions.parse and AsyncCompletions.parse with the same telemetry wrappers used for create(). ParsedChatCompletion extends ChatCompletion so the existing wrapper logic handles it correctly. A version guard (_is_parse_supported) skips wrapping on openai < 1.40.0 where parse() does not exist. response_format handling in both the legacy and experimental paths now recognises a bare Python type (Pydantic model class) and maps it to the appropriate output type attribute (json on experimental, json_schema on legacy). Includes sync and async tests with VCR cassettes, skipped on older SDK versions.
1 parent f1eb558 commit 8e8b16b

9 files changed

Lines changed: 882 additions & 4 deletions

File tree

‎instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,9 +97,25 @@
9797
)
9898

9999

100+
def _is_parse_supported():
101+
"""Check if the parse() method is available on the Completions class.
102+
103+
The parse() method for structured outputs was added in openai >= 1.40.0.
104+
"""
105+
try:
106+
from openai.resources.chat.completions import ( # pylint: disable=import-outside-toplevel # noqa: PLC0415
107+
Completions,
108+
)
109+
110+
return hasattr(Completions, "parse")
111+
except ImportError:
112+
return False
113+
114+
100115
class OpenAIInstrumentor(BaseInstrumentor):
101116
def __init__(self):
102117
self._meter = None
118+
self._parse_supported = False
103119

104120
def instrumentation_dependencies(self) -> Collection[str]:
105121
return _instruments
@@ -181,6 +197,36 @@ def _instrument(self, **kwargs):
181197
),
182198
)
183199

200+
# parse() wraps create() internally in the OpenAI SDK and returns a
201+
# ParsedChatCompletion. The telemetry-relevant fields (model, usage,
202+
# choices, finish_reason) are identical to ChatCompletion, so the
203+
# existing create() wrappers handle it correctly.
204+
self._parse_supported = _is_parse_supported()
205+
if self._parse_supported:
206+
wrap_function_wrapper(
207+
"openai.resources.chat.completions",
208+
"Completions.parse",
209+
(
210+
chat_completions_create_v_new(handler)
211+
if latest_experimental_enabled
212+
else chat_completions_create_v_old(
213+
tracer, logger, instruments, is_content_enabled()
214+
)
215+
),
216+
)
217+
218+
wrap_function_wrapper(
219+
"openai.resources.chat.completions",
220+
"AsyncCompletions.parse",
221+
(
222+
async_chat_completions_create_v_new(handler)
223+
if latest_experimental_enabled
224+
else async_chat_completions_create_v_old(
225+
tracer, logger, instruments, is_content_enabled()
226+
)
227+
),
228+
)
229+
184230
responses_module = _get_responses_module()
185231
# Responses instrumentation is intentionally limited to the latest
186232
# experimental semconv path. Unlike chat completions, we do not carry
@@ -201,6 +247,11 @@ def _uninstrument(self, **kwargs):
201247
unwrap(openai.resources.chat.completions.AsyncCompletions, "create")
202248
unwrap(openai.resources.embeddings.Embeddings, "create")
203249
unwrap(openai.resources.embeddings.AsyncEmbeddings, "create")
250+
if self._parse_supported:
251+
unwrap(openai.resources.chat.completions.Completions, "parse")
252+
unwrap(
253+
openai.resources.chat.completions.AsyncCompletions, "parse"
254+
)
204255
responses_module = _get_responses_module()
205256
if responses_module is not None:
206257
unwrap(responses_module.Responses, "create")

‎instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -280,8 +280,18 @@ def get_llm_request_attributes(
280280
else GenAIAttributes.GEN_AI_OPENAI_REQUEST_RESPONSE_FORMAT
281281
)
282282
if (response_format := kwargs.get("response_format")) is not None:
283-
# response_format may be string or object with a string in the `type` key
284-
if isinstance(response_format, Mapping):
283+
# response_format may be string, object with a string in the `type` key,
284+
# or a type (e.g. Pydantic model class used with parse())
285+
if isinstance(response_format, type):
286+
if latest_experimental_enabled:
287+
attributes[request_response_format_attr_key] = (
288+
GenAIAttributes.GenAiOutputTypeValues.JSON.value
289+
)
290+
else:
291+
attributes[request_response_format_attr_key] = (
292+
GenAIAttributes.GenAiOpenaiRequestResponseFormatValues.JSON_SCHEMA.value
293+
)
294+
elif isinstance(response_format, Mapping):
285295
if (
286296
response_format_type := response_format.get("type")
287297
) is not None:
@@ -369,8 +379,13 @@ def create_chat_invocation(
369379
if (
370380
response_format := get_value(kwargs.get("response_format"))
371381
) is not None:
372-
# response_format may be string or object with a string in the `type` key
373-
if isinstance(response_format, Mapping):
382+
# response_format may be string, object with a string in the `type` key,
383+
# or a type (e.g. Pydantic model class used with parse())
384+
if isinstance(response_format, type):
385+
invocation.attributes[GenAIAttributes.GEN_AI_OUTPUT_TYPE] = (
386+
GenAIAttributes.GenAiOutputTypeValues.JSON.value
387+
)
388+
elif isinstance(response_format, Mapping):
374389
if (
375390
response_format_type := get_value(response_format.get("type"))
376391
) is not None:

‎instrumentation/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_async_structured_output_no_content.yaml‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
interactions:
2+
- request:
3+
body: |-
4+
{
5+
"messages": [
6+
{
7+
"role": "user",
8+
"content": "Extract the event information from: Team Meeting on 2024-01-15 with Alice and Bob"
9+
}
10+
],
11+
"model": "gpt-4o-mini",
12+
"response_format": {
13+
"type": "json_schema",
14+
"json_schema": {
15+
"name": "CalendarEvent",
16+
"strict": true,
17+
"schema": {
18+
"type": "object",
19+
"properties": {
20+
"name": {"type": "string"},
21+
"date": {"type": "string"},
22+
"participants": {"items": {"type": "string"}, "type": "array"}
23+
},
24+
"required": ["name", "date", "participants"],
25+
"additionalProperties": false
26+
}
27+
}
28+
}
29+
}
30+
headers:
31+
accept:
32+
- application/json
33+
accept-encoding:
34+
- gzip, deflate
35+
authorization:
36+
- Bearer test_openai_api_key
37+
connection:
38+
- keep-alive
39+
content-type:
40+
- application/json
41+
host:
42+
- api.openai.com
43+
user-agent:
44+
- OpenAI/Python 1.54.3
45+
x-stainless-async:
46+
- async:asyncio
47+
x-stainless-lang:
48+
- python
49+
method: POST
50+
uri: https://api.openai.com/v1/chat/completions
51+
response:
52+
body:
53+
string: |-
54+
{
55+
"id": "chatcmpl-structured-test-004",
56+
"object": "chat.completion",
57+
"created": 1731368630,
58+
"model": "gpt-4o-mini-2024-07-18",
59+
"choices": [
60+
{
61+
"index": 0,
62+
"message": {
63+
"role": "assistant",
64+
"content": "{\"name\": \"Team Meeting\", \"date\": \"2024-01-15\", \"participants\": [\"Alice\", \"Bob\"]}",
65+
"refusal": null
66+
},
67+
"logprobs": null,
68+
"finish_reason": "stop"
69+
}
70+
],
71+
"usage": {
72+
"prompt_tokens": 50,
73+
"completion_tokens": 30,
74+
"total_tokens": 80,
75+
"prompt_tokens_details": {
76+
"cached_tokens": 0,
77+
"audio_tokens": 0
78+
},
79+
"completion_tokens_details": {
80+
"reasoning_tokens": 0,
81+
"audio_tokens": 0,
82+
"accepted_prediction_tokens": 0,
83+
"rejected_prediction_tokens": 0
84+
}
85+
},
86+
"system_fingerprint": "fp_0ba0d124f1"
87+
}
88+
headers:
89+
CF-Cache-Status:
90+
- DYNAMIC
91+
Connection:
92+
- keep-alive
93+
Content-Type:
94+
- application/json
95+
Date:
96+
- Mon, 11 Nov 2024 23:43:50 GMT
97+
Server:
98+
- cloudflare
99+
Set-Cookie: test_set_cookie
100+
Transfer-Encoding:
101+
- chunked
102+
X-Content-Type-Options:
103+
- nosniff
104+
content-length:
105+
- '800'
106+
openai-organization: test_openai_org_id
107+
openai-processing-ms:
108+
- '350'
109+
openai-version:
110+
- '2020-10-01'
111+
x-request-id:
112+
- req_structured_test_004
113+
status:
114+
code: 200
115+
message: OK
116+
version: 1

‎instrumentation/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_async_structured_output_with_content.yaml‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
interactions:
2+
- request:
3+
body: |-
4+
{
5+
"messages": [
6+
{
7+
"role": "user",
8+
"content": "Extract the event information from: Team Meeting on 2024-01-15 with Alice and Bob"
9+
}
10+
],
11+
"model": "gpt-4o-mini",
12+
"response_format": {
13+
"type": "json_schema",
14+
"json_schema": {
15+
"name": "CalendarEvent",
16+
"strict": true,
17+
"schema": {
18+
"type": "object",
19+
"properties": {
20+
"name": {"type": "string"},
21+
"date": {"type": "string"},
22+
"participants": {"items": {"type": "string"}, "type": "array"}
23+
},
24+
"required": ["name", "date", "participants"],
25+
"additionalProperties": false
26+
}
27+
}
28+
}
29+
}
30+
headers:
31+
accept:
32+
- application/json
33+
accept-encoding:
34+
- gzip, deflate
35+
authorization:
36+
- Bearer test_openai_api_key
37+
connection:
38+
- keep-alive
39+
content-type:
40+
- application/json
41+
host:
42+
- api.openai.com
43+
user-agent:
44+
- OpenAI/Python 1.54.3
45+
x-stainless-async:
46+
- async:asyncio
47+
x-stainless-lang:
48+
- python
49+
method: POST
50+
uri: https://api.openai.com/v1/chat/completions
51+
response:
52+
body:
53+
string: |-
54+
{
55+
"id": "chatcmpl-structured-test-003",
56+
"object": "chat.completion",
57+
"created": 1731368630,
58+
"model": "gpt-4o-mini-2024-07-18",
59+
"choices": [
60+
{
61+
"index": 0,
62+
"message": {
63+
"role": "assistant",
64+
"content": "{\"name\": \"Team Meeting\", \"date\": \"2024-01-15\", \"participants\": [\"Alice\", \"Bob\"]}",
65+
"refusal": null
66+
},
67+
"logprobs": null,
68+
"finish_reason": "stop"
69+
}
70+
],
71+
"usage": {
72+
"prompt_tokens": 50,
73+
"completion_tokens": 30,
74+
"total_tokens": 80,
75+
"prompt_tokens_details": {
76+
"cached_tokens": 0,
77+
"audio_tokens": 0
78+
},
79+
"completion_tokens_details": {
80+
"reasoning_tokens": 0,
81+
"audio_tokens": 0,
82+
"accepted_prediction_tokens": 0,
83+
"rejected_prediction_tokens": 0
84+
}
85+
},
86+
"system_fingerprint": "fp_0ba0d124f1"
87+
}
88+
headers:
89+
CF-Cache-Status:
90+
- DYNAMIC
91+
Connection:
92+
- keep-alive
93+
Content-Type:
94+
- application/json
95+
Date:
96+
- Mon, 11 Nov 2024 23:43:50 GMT
97+
Server:
98+
- cloudflare
99+
Set-Cookie: test_set_cookie
100+
Transfer-Encoding:
101+
- chunked
102+
X-Content-Type-Options:
103+
- nosniff
104+
content-length:
105+
- '800'
106+
openai-organization: test_openai_org_id
107+
openai-processing-ms:
108+
- '350'
109+
openai-version:
110+
- '2020-10-01'
111+
x-request-id:
112+
- req_structured_test_003
113+
status:
114+
code: 200
115+
message: OK
116+
version: 1

0 commit comments

Comments
 (0)