Skip to content

Commit 3786fa3

Browse files
committed
feat: skip response buffering for passthrough and internal model requests
Write SkipResponseBufferingKey to CycleState when response translation will be skipped, enabling real-time streaming instead of buffering the full response body: - Internal models (no ExternalModel match): no translation needed - Passthrough (inputFormat == outputFormat, not openai-chat): body passes through unmodified Depends on: llm-d/llm-d-inference-payload-processor#270
1 parent 7889f6a commit 3786fa3

1 file changed

Lines changed: 15 additions & 0 deletions

File tree

pkg/plugins/model-provider-resolver/plugin.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,7 @@ func (p *ModelProviderResolverPlugin) ProcessRequest(ctx context.Context, cycleS
163163
"original", modelName, "rewritten", parts[1])
164164
}
165165
}
166+
cycleState.Write(requesthandling.SkipResponseBufferingKey, true)
166167
return nil
167168
}
168169

@@ -199,10 +200,24 @@ func (p *ModelProviderResolverPlugin) ProcessRequest(ctx context.Context, cycleS
199200
cycleState.Write(state.ModelConfigKey, ref.config)
200201
cycleState.Write(state.InputAPIFormatKey, inputFormat)
201202

203+
if isPassthrough(inputFormat, apiformat.APIFormat(ref.apiFormat)) {
204+
cycleState.Write(requesthandling.SkipResponseBufferingKey, true)
205+
}
206+
202207
logger.Info("external model resolved", "model", modelName, "provider", ref.provider, "inputFormat", inputFormat, "apiFormat", ref.apiFormat)
203208
return nil
204209
}
205210

211+
func isPassthrough(inputFormat, outputFormat apiformat.APIFormat) bool {
212+
if inputFormat == "" || outputFormat == "" {
213+
return false
214+
}
215+
if inputFormat != outputFormat {
216+
return false
217+
}
218+
return inputFormat != apiformat.OpenAIChatCompletions
219+
}
220+
206221
// detectInputAPIFormat determines the client's API format from the request path suffix.
207222
func detectInputAPIFormat(path string) apiformat.APIFormat {
208223
switch {

0 commit comments

Comments
 (0)