Skip to content

Commit f73b822

Browse files
Nik-ReddyNik-Reddy
committed
Instrument OpenAI structured outputs (chat.completions.parse)
Add instrumentation for the parse() method on Completions and AsyncCompletions classes, which is used for OpenAI structured outputs. Changes: - __init__.py: Wrap/unwrap Completions.parse and AsyncCompletions.parse with a version guard (_is_parse_supported) for openai >= 1.40.0 - utils.py: Handle response_format as a type (Pydantic model class) by recording 'json_schema' as the output type attribute - Add sync and async test suites with VCR cassettes Closes #3449
1 parent 7f107df commit f73b822

8 files changed

Lines changed: 883 additions & 4 deletions

File tree

‎instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,21 @@
7272
)
7373

7474

75+
def _is_parse_supported():
76+
"""Check if the parse() method is available on the Completions class.
77+
78+
The parse() method for structured outputs was added in openai >= 1.40.0.
79+
"""
80+
try:
81+
from openai.resources.chat.completions import ( # pylint: disable=import-outside-toplevel # noqa: PLC0415
82+
Completions,
83+
)
84+
85+
return hasattr(Completions, "parse")
86+
except ImportError:
87+
return False
88+
89+
7590
class OpenAIInstrumentor(BaseInstrumentor):
7691
def __init__(self):
7792
self._meter = None
@@ -159,10 +174,39 @@ def _instrument(self, **kwargs):
159174
),
160175
)
161176

177+
if _is_parse_supported():
178+
wrap_function_wrapper(
179+
module="openai.resources.chat.completions",
180+
name="Completions.parse",
181+
wrapper=(
182+
chat_completions_create_v_new(handler, content_mode)
183+
if latest_experimental_enabled
184+
else chat_completions_create_v_old(
185+
tracer, logger, instruments, is_content_enabled()
186+
)
187+
),
188+
)
189+
190+
wrap_function_wrapper(
191+
module="openai.resources.chat.completions",
192+
name="AsyncCompletions.parse",
193+
wrapper=(
194+
async_chat_completions_create_v_new(handler, content_mode)
195+
if latest_experimental_enabled
196+
else async_chat_completions_create_v_old(
197+
tracer, logger, instruments, is_content_enabled()
198+
)
199+
),
200+
)
201+
162202
def _uninstrument(self, **kwargs):
163203
import openai # pylint: disable=import-outside-toplevel # noqa: PLC0415
164204

165205
unwrap(openai.resources.chat.completions.Completions, "create")
166206
unwrap(openai.resources.chat.completions.AsyncCompletions, "create")
167207
unwrap(openai.resources.embeddings.Embeddings, "create")
168208
unwrap(openai.resources.embeddings.AsyncEmbeddings, "create")
209+
210+
if _is_parse_supported():
211+
unwrap(openai.resources.chat.completions.Completions, "parse")
212+
unwrap(openai.resources.chat.completions.AsyncCompletions, "parse")

‎instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -275,8 +275,11 @@ def get_llm_request_attributes(
275275
else GenAIAttributes.GEN_AI_OPENAI_REQUEST_RESPONSE_FORMAT
276276
)
277277
if (response_format := kwargs.get("response_format")) is not None:
278-
# response_format may be string or object with a string in the `type` key
279-
if isinstance(response_format, Mapping):
278+
# response_format may be string, object with a string in the `type` key,
279+
# or a type (e.g. Pydantic model class used with parse())
280+
if isinstance(response_format, type):
281+
attributes[request_response_format_attr_key] = "json_schema"
282+
elif isinstance(response_format, Mapping):
280283
if (
281284
response_format_type := response_format.get("type")
282285
) is not None:
@@ -369,8 +372,11 @@ def create_chat_invocation(
369372
if (
370373
response_format := get_value(kwargs.get("response_format"))
371374
) is not None:
372-
# response_format may be string or object with a string in the `type` key
373-
if isinstance(response_format, Mapping):
375+
# response_format may be string, object with a string in the `type` key,
376+
# or a type (e.g. Pydantic model class used with parse())
377+
if isinstance(response_format, type):
378+
attributes[GenAIAttributes.GEN_AI_OUTPUT_TYPE] = "json_schema"
379+
elif isinstance(response_format, Mapping):
374380
if (
375381
response_format_type := get_value(response_format.get("type"))
376382
) is not None:

‎instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_async_structured_output_no_content.yaml‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
interactions:
2+
- request:
3+
body: |-
4+
{
5+
"messages": [
6+
{
7+
"role": "user",
8+
"content": "Extract the event information from: Team Meeting on 2024-01-15 with Alice and Bob"
9+
}
10+
],
11+
"model": "gpt-4o-mini",
12+
"response_format": {
13+
"type": "json_schema",
14+
"json_schema": {
15+
"name": "CalendarEvent",
16+
"strict": true,
17+
"schema": {
18+
"type": "object",
19+
"properties": {
20+
"name": {"type": "string"},
21+
"date": {"type": "string"},
22+
"participants": {"items": {"type": "string"}, "type": "array"}
23+
},
24+
"required": ["name", "date", "participants"],
25+
"additionalProperties": false
26+
}
27+
}
28+
}
29+
}
30+
headers:
31+
accept:
32+
- application/json
33+
accept-encoding:
34+
- gzip, deflate
35+
authorization:
36+
- Bearer test_openai_api_key
37+
connection:
38+
- keep-alive
39+
content-type:
40+
- application/json
41+
host:
42+
- api.openai.com
43+
user-agent:
44+
- OpenAI/Python 1.54.3
45+
x-stainless-async:
46+
- async:asyncio
47+
x-stainless-lang:
48+
- python
49+
method: POST
50+
uri: https://api.openai.com/v1/chat/completions
51+
response:
52+
body:
53+
string: |-
54+
{
55+
"id": "chatcmpl-structured-test-004",
56+
"object": "chat.completion",
57+
"created": 1731368630,
58+
"model": "gpt-4o-mini-2024-07-18",
59+
"choices": [
60+
{
61+
"index": 0,
62+
"message": {
63+
"role": "assistant",
64+
"content": "{\"name\": \"Team Meeting\", \"date\": \"2024-01-15\", \"participants\": [\"Alice\", \"Bob\"]}",
65+
"refusal": null
66+
},
67+
"logprobs": null,
68+
"finish_reason": "stop"
69+
}
70+
],
71+
"usage": {
72+
"prompt_tokens": 50,
73+
"completion_tokens": 30,
74+
"total_tokens": 80,
75+
"prompt_tokens_details": {
76+
"cached_tokens": 0,
77+
"audio_tokens": 0
78+
},
79+
"completion_tokens_details": {
80+
"reasoning_tokens": 0,
81+
"audio_tokens": 0,
82+
"accepted_prediction_tokens": 0,
83+
"rejected_prediction_tokens": 0
84+
}
85+
},
86+
"system_fingerprint": "fp_0ba0d124f1"
87+
}
88+
headers:
89+
CF-Cache-Status:
90+
- DYNAMIC
91+
Connection:
92+
- keep-alive
93+
Content-Type:
94+
- application/json
95+
Date:
96+
- Mon, 11 Nov 2024 23:43:50 GMT
97+
Server:
98+
- cloudflare
99+
Set-Cookie: test_set_cookie
100+
Transfer-Encoding:
101+
- chunked
102+
X-Content-Type-Options:
103+
- nosniff
104+
content-length:
105+
- '800'
106+
openai-organization: test_openai_org_id
107+
openai-processing-ms:
108+
- '350'
109+
openai-version:
110+
- '2020-10-01'
111+
x-request-id:
112+
- req_structured_test_004
113+
status:
114+
code: 200
115+
message: OK
116+
version: 1

‎instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_async_structured_output_with_content.yaml‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
interactions:
2+
- request:
3+
body: |-
4+
{
5+
"messages": [
6+
{
7+
"role": "user",
8+
"content": "Extract the event information from: Team Meeting on 2024-01-15 with Alice and Bob"
9+
}
10+
],
11+
"model": "gpt-4o-mini",
12+
"response_format": {
13+
"type": "json_schema",
14+
"json_schema": {
15+
"name": "CalendarEvent",
16+
"strict": true,
17+
"schema": {
18+
"type": "object",
19+
"properties": {
20+
"name": {"type": "string"},
21+
"date": {"type": "string"},
22+
"participants": {"items": {"type": "string"}, "type": "array"}
23+
},
24+
"required": ["name", "date", "participants"],
25+
"additionalProperties": false
26+
}
27+
}
28+
}
29+
}
30+
headers:
31+
accept:
32+
- application/json
33+
accept-encoding:
34+
- gzip, deflate
35+
authorization:
36+
- Bearer test_openai_api_key
37+
connection:
38+
- keep-alive
39+
content-type:
40+
- application/json
41+
host:
42+
- api.openai.com
43+
user-agent:
44+
- OpenAI/Python 1.54.3
45+
x-stainless-async:
46+
- async:asyncio
47+
x-stainless-lang:
48+
- python
49+
method: POST
50+
uri: https://api.openai.com/v1/chat/completions
51+
response:
52+
body:
53+
string: |-
54+
{
55+
"id": "chatcmpl-structured-test-003",
56+
"object": "chat.completion",
57+
"created": 1731368630,
58+
"model": "gpt-4o-mini-2024-07-18",
59+
"choices": [
60+
{
61+
"index": 0,
62+
"message": {
63+
"role": "assistant",
64+
"content": "{\"name\": \"Team Meeting\", \"date\": \"2024-01-15\", \"participants\": [\"Alice\", \"Bob\"]}",
65+
"refusal": null
66+
},
67+
"logprobs": null,
68+
"finish_reason": "stop"
69+
}
70+
],
71+
"usage": {
72+
"prompt_tokens": 50,
73+
"completion_tokens": 30,
74+
"total_tokens": 80,
75+
"prompt_tokens_details": {
76+
"cached_tokens": 0,
77+
"audio_tokens": 0
78+
},
79+
"completion_tokens_details": {
80+
"reasoning_tokens": 0,
81+
"audio_tokens": 0,
82+
"accepted_prediction_tokens": 0,
83+
"rejected_prediction_tokens": 0
84+
}
85+
},
86+
"system_fingerprint": "fp_0ba0d124f1"
87+
}
88+
headers:
89+
CF-Cache-Status:
90+
- DYNAMIC
91+
Connection:
92+
- keep-alive
93+
Content-Type:
94+
- application/json
95+
Date:
96+
- Mon, 11 Nov 2024 23:43:50 GMT
97+
Server:
98+
- cloudflare
99+
Set-Cookie: test_set_cookie
100+
Transfer-Encoding:
101+
- chunked
102+
X-Content-Type-Options:
103+
- nosniff
104+
content-length:
105+
- '800'
106+
openai-organization: test_openai_org_id
107+
openai-processing-ms:
108+
- '350'
109+
openai-version:
110+
- '2020-10-01'
111+
x-request-id:
112+
- req_structured_test_003
113+
status:
114+
code: 200
115+
message: OK
116+
version: 1

0 commit comments

Comments
 (0)