Skip to content

Commit 9f17227

Browse files
Copilotwaldekmastykarzgarrytrinder
authored
Add support for OpenAI Responses API (#1492)
* Initial plan * Add OpenAI Responses API support to all OpenAI-related plugins Co-authored-by: waldekmastykarz <11164679+waldekmastykarz@users.noreply.github.com> * Address code review: reduce code duplication, add constant for message ID prefix Co-authored-by: waldekmastykarz <11164679+waldekmastykarz@users.noreply.github.com> * Improve documentation for code clarity Co-authored-by: waldekmastykarz <11164679+waldekmastykarz@users.noreply.github.com> * Address review comments - Extract shared IsResponsesApiRequest helper to deduplicate detection logic - Remove unused OpenAIResponsesUsage class - Add missing base properties in LanguageModelFailurePlugin Responses API branch - Fix complex content formatting in OpenAITelemetryPlugin --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: waldekmastykarz <11164679+waldekmastykarz@users.noreply.github.com> Co-authored-by: Waldek Mastykarz <waldek@mastykarz.nl> Co-authored-by: Garry Trinder <garry@trinder365.co.uk>
1 parent a5cc723 commit 9f17227

6 files changed

Lines changed: 326 additions & 113 deletions

File tree

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -349,4 +349,5 @@ MigrationBackup/
349349
# Ionide (cross platform F# VS Code tools) working folder
350350
.ionide/
351351

352-
.DS_Store
352+
.DS_Store
353+
.tmp/

‎DevProxy.Abstractions/LanguageModel/OpenAIModels.cs‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,15 @@ public static bool TryGetOpenAIRequest(string content, ILogger logger, out OpenA
6060
return true;
6161
}
6262

63+
// Responses API request - has "input" array with objects containing role/content
64+
// Must be checked before embedding request because both have "input"
65+
if (IsResponsesApiRequest(rawRequest))
66+
{
67+
logger.LogDebug("Request is a Responses API request");
68+
request = JsonSerializer.Deserialize<OpenAIResponsesRequest>(content, ProxyUtils.JsonSerializerOptions);
69+
return true;
70+
}
71+
6372
// Embedding request
6473
if (rawRequest.TryGetProperty("input", out _) &&
6574
rawRequest.TryGetProperty("model", out _) &&
@@ -112,6 +121,73 @@ public static bool TryGetOpenAIRequest(string content, ILogger logger, out OpenA
112121
return false;
113122
}
114123
}
124+
125+
/// <summary>
126+
/// Tries to parse text generation OpenAI requests (completion, chat completion, and responses API).
127+
/// Used by plugins that only need to handle text-based generation requests, as opposed to
128+
/// embeddings, audio, images, or fine-tuning requests.
129+
/// </summary>
130+
public static bool TryGetCompletionLikeRequest(string content, ILogger logger, out OpenAIRequest? request)
131+
{
132+
logger.LogTrace("{Method} called", nameof(TryGetCompletionLikeRequest));
133+
134+
request = null;
135+
136+
if (string.IsNullOrEmpty(content))
137+
{
138+
logger.LogDebug("Request content is empty or null");
139+
return false;
140+
}
141+
142+
try
143+
{
144+
logger.LogDebug("Checking if the request is a completion-like OpenAI request...");
145+
146+
var rawRequest = JsonSerializer.Deserialize<JsonElement>(content, ProxyUtils.JsonSerializerOptions);
147+
148+
// Completion request
149+
if (rawRequest.TryGetProperty("prompt", out _))
150+
{
151+
logger.LogDebug("Request is a completion request");
152+
request = JsonSerializer.Deserialize<OpenAICompletionRequest>(content, ProxyUtils.JsonSerializerOptions);
153+
return true;
154+
}
155+
156+
// Chat completion request
157+
if (rawRequest.TryGetProperty("messages", out _))
158+
{
159+
logger.LogDebug("Request is a chat completion request");
160+
request = JsonSerializer.Deserialize<OpenAIChatCompletionRequest>(content, ProxyUtils.JsonSerializerOptions);
161+
return true;
162+
}
163+
164+
// Responses API request - has "input" array with objects containing role/content
165+
if (IsResponsesApiRequest(rawRequest))
166+
{
167+
logger.LogDebug("Request is a Responses API request");
168+
request = JsonSerializer.Deserialize<OpenAIResponsesRequest>(content, ProxyUtils.JsonSerializerOptions);
169+
return true;
170+
}
171+
172+
logger.LogDebug("Request is not a completion-like OpenAI request.");
173+
return false;
174+
}
175+
catch (JsonException ex)
176+
{
177+
logger.LogDebug(ex, "Failed to deserialize OpenAI request.");
178+
return false;
179+
}
180+
}
181+
182+
private static bool IsResponsesApiRequest(JsonElement rawRequest)
183+
{
184+
return rawRequest.TryGetProperty("input", out var inputProperty) &&
185+
inputProperty.ValueKind == JsonValueKind.Array &&
186+
inputProperty.GetArrayLength() > 0 &&
187+
inputProperty.EnumerateArray().First().ValueKind == JsonValueKind.Object &&
188+
(inputProperty.EnumerateArray().First().TryGetProperty("role", out _) ||
189+
inputProperty.EnumerateArray().First().TryGetProperty("type", out _));
190+
}
115191
}
116192

117193
public class OpenAIResponse : ILanguageModelCompletionResponse
@@ -178,6 +254,23 @@ public class OpenAIResponseUsage
178254
public PromptTokenDetails? PromptTokensDetails { get; set; }
179255
[JsonPropertyName("total_tokens")]
180256
public long TotalTokens { get; set; }
257+
258+
// Responses API uses different property names (input_tokens, output_tokens)
259+
// These property aliases allow the same class to deserialize both formats.
260+
// When JSON contains "input_tokens", it maps to PromptTokens.
261+
// When JSON contains "output_tokens", it maps to CompletionTokens.
262+
[JsonPropertyName("input_tokens")]
263+
public long InputTokens
264+
{
265+
get => PromptTokens;
266+
set => PromptTokens = value;
267+
}
268+
[JsonPropertyName("output_tokens")]
269+
public long OutputTokens
270+
{
271+
get => CompletionTokens;
272+
set => CompletionTokens = value;
273+
}
181274
}
182275

183276
public class PromptTokenDetails
@@ -409,3 +502,71 @@ public class OpenAIImageData
409502
[JsonPropertyName("revised_prompt")]
410503
public string? RevisedPrompt { get; set; }
411504
}
505+
506+
#region Responses API
507+
508+
public class OpenAIResponsesRequest : OpenAIRequest
509+
{
510+
public IEnumerable<OpenAIResponsesInputItem>? Input { get; set; }
511+
public string? Instructions { get; set; }
512+
[JsonPropertyName("previous_response_id")]
513+
public string? PreviousResponseId { get; set; }
514+
[JsonPropertyName("max_output_tokens")]
515+
public long? MaxOutputTokens { get; set; }
516+
}
517+
518+
public class OpenAIResponsesInputItem
519+
{
520+
public string Role { get; set; } = string.Empty;
521+
[JsonConverter(typeof(OpenAIContentPartJsonConverter))]
522+
public object Content { get; set; } = string.Empty;
523+
public string? Type { get; set; }
524+
}
525+
526+
public class OpenAIResponsesResponse : OpenAIResponse
527+
{
528+
[JsonPropertyName("created_at")]
529+
public long CreatedAt { get; set; }
530+
public string Status { get; set; } = string.Empty;
531+
public IEnumerable<OpenAIResponsesOutputItem>? Output { get; set; }
532+
[JsonPropertyName("previous_response_id")]
533+
public string? PreviousResponseId { get; set; }
534+
535+
public override string? Response => GetTextFromOutput();
536+
537+
private string? GetTextFromOutput()
538+
{
539+
if (Output is null)
540+
{
541+
return null;
542+
}
543+
544+
var messageOutput = Output.FirstOrDefault(o =>
545+
string.Equals(o.Type, "message", StringComparison.OrdinalIgnoreCase));
546+
if (messageOutput?.Content is null)
547+
{
548+
return null;
549+
}
550+
551+
var textContent = messageOutput.Content.FirstOrDefault(c =>
552+
string.Equals(c.Type, "output_text", StringComparison.OrdinalIgnoreCase));
553+
return textContent?.Text;
554+
}
555+
}
556+
557+
public class OpenAIResponsesOutputItem
558+
{
559+
public string? Type { get; set; }
560+
public string? Id { get; set; }
561+
public string? Role { get; set; }
562+
public IEnumerable<OpenAIResponsesOutputContent>? Content { get; set; }
563+
public string? Status { get; set; }
564+
}
565+
566+
public class OpenAIResponsesOutputContent
567+
{
568+
public string? Type { get; set; }
569+
public string? Text { get; set; }
570+
}
571+
572+
#endregion

‎DevProxy.Plugins/Behavior/LanguageModelFailurePlugin.cs‎

Lines changed: 31 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ public override async Task BeforeRequestAsync(ProxyRequestArgs e, CancellationTo
7373
return;
7474
}
7575

76-
if (!TryGetOpenAIRequest(request.BodyString, out var openAiRequest))
76+
if (!OpenAIRequest.TryGetCompletionLikeRequest(request.BodyString, Logger, out var openAiRequest))
7777
{
7878
Logger.LogRequest("Skipping non-OpenAI request", MessageType.Skipped, new LoggingContext(e.Session));
7979
return;
@@ -116,6 +116,36 @@ public override async Task BeforeRequestAsync(ProxyRequestArgs e, CancellationTo
116116
Logger.LogRequest($"Simulating fault {faultName}", MessageType.Chaos, new LoggingContext(e.Session));
117117
e.Session.SetRequestBodyString(JsonSerializer.Serialize(newRequest, ProxyUtils.JsonSerializerOptions));
118118
}
119+
else if (openAiRequest is OpenAIResponsesRequest responsesRequest)
120+
{
121+
var inputItems = new List<OpenAIResponsesInputItem>(responsesRequest.Input ?? [])
122+
{
123+
new()
124+
{
125+
Role = "user",
126+
Content = faultPrompt
127+
}
128+
};
129+
var newRequest = new OpenAIResponsesRequest
130+
{
131+
Model = responsesRequest.Model,
132+
Stream = responsesRequest.Stream,
133+
Temperature = responsesRequest.Temperature,
134+
TopP = responsesRequest.TopP,
135+
FrequencyPenalty = responsesRequest.FrequencyPenalty,
136+
MaxTokens = responsesRequest.MaxTokens,
137+
PresencePenalty = responsesRequest.PresencePenalty,
138+
Stop = responsesRequest.Stop,
139+
Instructions = responsesRequest.Instructions,
140+
PreviousResponseId = responsesRequest.PreviousResponseId,
141+
MaxOutputTokens = responsesRequest.MaxOutputTokens,
142+
Input = inputItems
143+
};
144+
145+
Logger.LogDebug("Added fault prompt to Responses API input: {Prompt}", faultPrompt);
146+
Logger.LogRequest($"Simulating fault {faultName}", MessageType.Chaos, new LoggingContext(e.Session));
147+
e.Session.SetRequestBodyString(JsonSerializer.Serialize(newRequest, ProxyUtils.JsonSerializerOptions));
148+
}
119149
else
120150
{
121151
Logger.LogDebug("Unknown OpenAI request type. Passing request as-is.");
@@ -126,45 +156,6 @@ public override async Task BeforeRequestAsync(ProxyRequestArgs e, CancellationTo
126156
Logger.LogTrace("Left {Name}", nameof(BeforeRequestAsync));
127157
}
128158

129-
private bool TryGetOpenAIRequest(string content, out OpenAIRequest? request)
130-
{
131-
request = null;
132-
133-
if (string.IsNullOrEmpty(content))
134-
{
135-
return false;
136-
}
137-
138-
try
139-
{
140-
Logger.LogDebug("Checking if the request is an OpenAI request...");
141-
142-
var rawRequest = JsonSerializer.Deserialize<JsonElement>(content, ProxyUtils.JsonSerializerOptions);
143-
144-
if (rawRequest.TryGetProperty("prompt", out _))
145-
{
146-
Logger.LogDebug("Request is a completion request");
147-
request = JsonSerializer.Deserialize<OpenAICompletionRequest>(content, ProxyUtils.JsonSerializerOptions);
148-
return true;
149-
}
150-
151-
if (rawRequest.TryGetProperty("messages", out _))
152-
{
153-
Logger.LogDebug("Request is a chat completion request");
154-
request = JsonSerializer.Deserialize<OpenAIChatCompletionRequest>(content, ProxyUtils.JsonSerializerOptions);
155-
return true;
156-
}
157-
158-
Logger.LogDebug("Request is not an OpenAI request.");
159-
return false;
160-
}
161-
catch (JsonException ex)
162-
{
163-
Logger.LogDebug(ex, "Failed to deserialize OpenAI request.");
164-
return false;
165-
}
166-
}
167-
168159
private (string? Name, string? Prompt) GetFault()
169160
{
170161
var failures = Configuration.Failures?.ToArray() ?? _defaultFailures;

‎DevProxy.Plugins/Behavior/LanguageModelRateLimitingPlugin.cs‎

Lines changed: 2 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ public override Task BeforeRequestAsync(ProxyRequestArgs e, CancellationToken ca
9999
return Task.CompletedTask;
100100
}
101101

102-
if (!TryGetOpenAIRequest(request.BodyString, out var openAiRequest))
102+
if (!OpenAIRequest.TryGetCompletionLikeRequest(request.BodyString, Logger, out var openAiRequest))
103103
{
104104
Logger.LogRequest("Skipping non-OpenAI request", MessageType.Skipped, new LoggingContext(e.Session));
105105
return Task.CompletedTask;
@@ -224,7 +224,7 @@ public override Task BeforeResponseAsync(ProxyResponseArgs e, CancellationToken
224224
return Task.CompletedTask;
225225
}
226226

227-
if (!TryGetOpenAIRequest(request.BodyString, out var openAiRequest))
227+
if (!OpenAIRequest.TryGetCompletionLikeRequest(request.BodyString, Logger, out var openAiRequest))
228228
{
229229
Logger.LogDebug("Skipping non-OpenAI request");
230230
return Task.CompletedTask;
@@ -271,45 +271,6 @@ public override Task BeforeResponseAsync(ProxyResponseArgs e, CancellationToken
271271
return Task.CompletedTask;
272272
}
273273

274-
private bool TryGetOpenAIRequest(string content, out OpenAIRequest? request)
275-
{
276-
request = null;
277-
278-
if (string.IsNullOrEmpty(content))
279-
{
280-
return false;
281-
}
282-
283-
try
284-
{
285-
Logger.LogDebug("Checking if the request is an OpenAI request...");
286-
287-
var rawRequest = JsonSerializer.Deserialize<JsonElement>(content, ProxyUtils.JsonSerializerOptions);
288-
289-
if (rawRequest.TryGetProperty("prompt", out _))
290-
{
291-
Logger.LogDebug("Request is a completion request");
292-
request = JsonSerializer.Deserialize<OpenAICompletionRequest>(content, ProxyUtils.JsonSerializerOptions);
293-
return true;
294-
}
295-
296-
if (rawRequest.TryGetProperty("messages", out _))
297-
{
298-
Logger.LogDebug("Request is a chat completion request");
299-
request = JsonSerializer.Deserialize<OpenAIChatCompletionRequest>(content, ProxyUtils.JsonSerializerOptions);
300-
return true;
301-
}
302-
303-
Logger.LogDebug("Request is not an OpenAI request.");
304-
return false;
305-
}
306-
catch (JsonException ex)
307-
{
308-
Logger.LogDebug(ex, "Failed to deserialize OpenAI request.");
309-
return false;
310-
}
311-
}
312-
313274
private ThrottlingInfo ShouldThrottle(Request request, string throttlingKey)
314275
{
315276
var throttleKeyForRequest = BuildThrottleKey(request);

0 commit comments

Comments
 (0)