Skip to content

Commit f8a8947

Browse files
ssncferreirassncferreira
authored
fix: handle chatcompletions streaming tool calls with no text preamble and non-zero indices (#147)
## Description Fixes two edge cases in streaming chat completions when handling injected tool calls observed with Copilot Provider: * Tool call as first chunk (no preamble): When the provider responds with only a tool call and no text content, `events.IsStreaming()` returns false, causing the request to end prematurely without invoking the tool. * Tool call with non-zero index: When the provider returns a tool call at `index: 1` instead of `index: 0`, the OpenAI SDK accumulator creates `[nil, {tool}]`. When this is appended to the messages for the next request, the provider rejects it. ## Changes * Add `MarkInitiated()` to `EventStream` to mark the stream as active when processing injected tool calls that don't relay chunks to the client * Add `compactToolCalls()` to remove nil/empty entries from the tool calls array before appending completions to messages * Add integration tests with fixtures for both edge cases
1 parent f603a56 commit f8a8947

6 files changed

Lines changed: 343 additions & 44 deletions

File tree

‎bridge_integration_test.go‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -438,6 +438,119 @@ func TestOpenAIChatCompletions(t *testing.T) {
438438
})
439439
}
440440
})
441+
442+
t.Run("streaming injected tool call edge cases", func(t *testing.T) {
443+
t.Parallel()
444+
445+
cases := []struct {
446+
name string
447+
fixture []byte
448+
expectedArgs map[string]any
449+
}{
450+
{
451+
name: "tool call no preamble",
452+
fixture: fixtures.OaiChatStreamingInjectedToolNoPreamble,
453+
expectedArgs: map[string]any{"owner": "me"},
454+
},
455+
{
456+
name: "tool call with non-zero index",
457+
fixture: fixtures.OaiChatStreamingInjectedToolNonzeroIndex,
458+
expectedArgs: nil, // No arguments in this fixture
459+
},
460+
}
461+
462+
for _, tc := range cases {
463+
t.Run(tc.name, func(t *testing.T) {
464+
t.Parallel()
465+
466+
arc := txtar.Parse(tc.fixture)
467+
t.Logf("%s: %s", t.Name(), arc.Comment)
468+
469+
files := filesMap(arc)
470+
require.Len(t, files, 3)
471+
require.Contains(t, files, fixtureRequest)
472+
require.Contains(t, files, fixtureStreamingResponse)
473+
require.Contains(t, files, fixtureStreamingToolResponse)
474+
475+
reqBody := files[fixtureRequest]
476+
477+
// Add the stream param to the request.
478+
newBody, err := setJSON(reqBody, "stream", true)
479+
require.NoError(t, err)
480+
reqBody = newBody
481+
482+
ctx, cancel := context.WithTimeout(t.Context(), time.Second*30)
483+
t.Cleanup(cancel)
484+
485+
// Setup mock server with response mutator for multi-turn interaction.
486+
srv := newMockServer(ctx, t, files, func(reqCount uint32, resp []byte) []byte {
487+
if reqCount == 1 {
488+
// First request gets the tool call response
489+
return resp
490+
}
491+
// Second request gets final response
492+
return files[fixtureStreamingToolResponse]
493+
})
494+
t.Cleanup(srv.Close)
495+
496+
recorderClient := &testutil.MockRecorder{}
497+
498+
// Setup MCP proxies with the tool from the fixture
499+
mcpProxiers, mcpCalls := setupMCPServerProxiesForTest(t, testTracer)
500+
mcpMgr := mcp.NewServerProxyManager(mcpProxiers, testTracer)
501+
require.NoError(t, mcpMgr.Init(ctx))
502+
503+
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: false}).Leveled(slog.LevelDebug)
504+
providers := []aibridge.Provider{provider.NewOpenAI(openaiCfg(srv.URL, apiKey))}
505+
b, err := aibridge.NewRequestBridge(t.Context(), providers, recorderClient, mcpMgr, logger, nil, testTracer)
506+
require.NoError(t, err)
507+
508+
mockSrv := httptest.NewUnstartedServer(b)
509+
t.Cleanup(mockSrv.Close)
510+
mockSrv.Config.BaseContext = func(_ net.Listener) context.Context {
511+
return aibcontext.AsActor(ctx, userID, nil)
512+
}
513+
mockSrv.Start()
514+
515+
req := createOpenAIChatCompletionsReq(t, mockSrv.URL, reqBody)
516+
517+
client := &http.Client{}
518+
resp, err := client.Do(req)
519+
require.NoError(t, err)
520+
require.Equal(t, http.StatusOK, resp.StatusCode)
521+
522+
// Verify SSE headers are sent correctly
523+
require.Equal(t, "text/event-stream", resp.Header.Get("Content-Type"))
524+
require.Equal(t, "no-cache", resp.Header.Get("Cache-Control"))
525+
require.Equal(t, "keep-alive", resp.Header.Get("Connection"))
526+
527+
// Consume the full response body to ensure the interception completes
528+
_, err = io.ReadAll(resp.Body)
529+
require.NoError(t, err)
530+
resp.Body.Close()
531+
532+
// Verify the MCP tool was actually invoked
533+
invocations := mcpCalls.getCallsByTool(mockToolName)
534+
require.Len(t, invocations, 1, "expected MCP tool to be invoked")
535+
536+
// Verify tool was invoked with the expected args (if specified)
537+
if tc.expectedArgs != nil {
538+
expected, err := json.Marshal(tc.expectedArgs)
539+
require.NoError(t, err)
540+
actual, err := json.Marshal(invocations[0])
541+
require.NoError(t, err)
542+
require.EqualValues(t, expected, actual)
543+
}
544+
545+
// Verify tool usage was recorded
546+
toolUsages := recorderClient.RecordedToolUsages()
547+
require.Len(t, toolUsages, 1)
548+
assert.Equal(t, mockToolName, toolUsages[0].Tool)
549+
550+
recorderClient.VerifyAllInterceptionsEnded(t)
551+
})
552+
}
553+
})
441554
}
442555

443556
func TestSimple(t *testing.T) {

‎fixtures/fixtures.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,12 @@ var (
4545

4646
//go:embed openai/chatcompletions/non_stream_error.txtar
4747
OaiChatNonStreamError []byte
48+
49+
//go:embed openai/chatcompletions/streaming_injected_tool_no_preamble.txtar
50+
OaiChatStreamingInjectedToolNoPreamble []byte
51+
52+
//go:embed openai/chatcompletions/streaming_injected_tool_nonzero_index.txtar
53+
OaiChatStreamingInjectedToolNonzeroIndex []byte
4854
)
4955

5056
var (

‎fixtures/openai/chatcompletions/streaming_injected_tool_no_preamble.txtar‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
Streaming response where the provider returns an injected tool call as the first chunk with no text preamble.
2+
This test ensures tool invocation continues even when no chunks are relayed to the client.
3+
4+
-- request --
5+
{
6+
"messages": [
7+
{
8+
"content": "<current_datetime>2026-01-22T18:35:17.612Z</current_datetime>\n\nlist all my coder workspaces",
9+
"role": "user"
10+
}
11+
],
12+
"model": "claude-haiku-4.5",
13+
"n": 1,
14+
"temperature": 1,
15+
"parallel_tool_calls": false,
16+
"stream_options": {
17+
"include_usage": true
18+
},
19+
"stream": true
20+
}
21+
22+
-- streaming --
23+
data: {"choices":[{"index":0,"delta":{"content":null,"tool_calls":[{"function":{"name":"bmcp_coder_coder_list_workspaces"},"id":"toolu_vrtx_01CvBi1d4qpKTG2PCuc9wDbZ","index":0,"type":"function"}]}}],"created":1769106921,"id":"msg_vrtx_01UoiRJwj3JXcwNYAh3z7ARs","model":"claude-haiku-4.5"}
24+
25+
data: {"choices":[{"index":0,"delta":{"content":null,"tool_calls":[{"function":{"arguments":""},"index":0}]}}],"created":1769106921,"id":"msg_vrtx_01UoiRJwj3JXcwNYAh3z7ARs","model":"claude-haiku-4.5"}
26+
27+
data: {"choices":[{"index":0,"delta":{"content":null,"tool_calls":[{"function":{"arguments":"{\"own"},"index":0}]}}],"created":1769106921,"id":"msg_vrtx_01UoiRJwj3JXcwNYAh3z7ARs","model":"claude-haiku-4.5"}
28+
29+
data: {"choices":[{"index":0,"delta":{"content":null,"tool_calls":[{"function":{"arguments":"er\": \"me\"}"},"index":0}]}}],"created":1769106921,"id":"msg_vrtx_01UoiRJwj3JXcwNYAh3z7ARs","model":"claude-haiku-4.5"}
30+
31+
data: {"choices":[{"finish_reason":"tool_calls","index":0,"delta":{"content":null}}],"created":1769106921,"id":"msg_vrtx_01UoiRJwj3JXcwNYAh3z7ARs","usage":{"completion_tokens":65,"prompt_tokens":25716,"prompt_tokens_details":{"cached_tokens":20470},"total_tokens":25781},"model":"claude-haiku-4.5"}
32+
33+
data: [DONE]
34+
35+
36+
-- streaming/tool-call --
37+
data: {"choices":[{"index":0,"delta":{"content":"You","role":"assistant"}}],"created":1769198061,"id":"msg_vrtx_015B1npskreQgEjMrfsdjH1m","model":"claude-haiku-4.5"}
38+
39+
data: {"choices":[{"index":0,"delta":{"content":" have one","role":"assistant"}}],"created":1769198061,"id":"msg_vrtx_015B1npskreQgEjMrfsdjH1m","model":"claude-haiku-4.5"}
40+
41+
data: {"choices":[{"index":0,"delta":{"content":" Coder workspace:","role":"assistant"}}],"created":1769198062,"id":"msg_vrtx_015B1npskreQgEjMrfsdjH1m","model":"claude-haiku-4.5"}
42+
43+
data: {"choices":[{"index":0,"delta":{"content":"\n\n**test-scf** (","role":"assistant"}}],"created":1769198062,"id":"msg_vrtx_015B1npskreQgEjMrfsdjH1m","model":"claude-haiku-4.5"}
44+
45+
data: {"choices":[{"index":0,"delta":{"content":"ID: a174a2e5","role":"assistant"}}],"created":1769198062,"id":"msg_vrtx_015B1npskreQgEjMrfsdjH1m","model":"claude-haiku-4.5"}
46+
47+
data: {"choices":[{"index":0,"delta":{"content":"-5050-445d-89","role":"assistant"}}],"created":1769198062,"id":"msg_vrtx_015B1npskreQgEjMrfsdjH1m","model":"claude-haiku-4.5"}
48+
49+
data: {"choices":[{"index":0,"delta":{"content":"ff-dd720e5b442","role":"assistant"}}],"created":1769198062,"id":"msg_vrtx_015B1npskreQgEjMrfsdjH1m","model":"claude-haiku-4.5"}
50+
51+
data: {"choices":[{"index":0,"delta":{"content":"e)\n- Template: docker","role":"assistant"}}],"created":1769198062,"id":"msg_vrtx_015B1npskreQgEjMrfsdjH1m","model":"claude-haiku-4.5"}
52+
53+
data: {"choices":[{"index":0,"delta":{"content":"\n- Template Version","role":"assistant"}}],"created":1769198062,"id":"msg_vrtx_015B1npskreQgEjMrfsdjH1m","model":"claude-haiku-4.5"}
54+
55+
data: {"choices":[{"index":0,"delta":{"content":" ID","role":"assistant"}}],"created":1769198062,"id":"msg_vrtx_015B1npskreQgEjMrfsdjH1m","model":"claude-haiku-4.5"}
56+
57+
data: {"choices":[{"index":0,"delta":{"content":": ad1b5ab1-","role":"assistant"}}],"created":1769198062,"id":"msg_vrtx_015B1npskreQgEjMrfsdjH1m","model":"claude-haiku-4.5"}
58+
59+
data: {"choices":[{"index":0,"delta":{"content":"fc18-4792-84f","role":"assistant"}}],"created":1769198062,"id":"msg_vrtx_015B1npskreQgEjMrfsdjH1m","model":"claude-haiku-4.5"}
60+
61+
data: {"choices":[{"index":0,"delta":{"content":"7-797787607d30","role":"assistant"}}],"created":1769198062,"id":"msg_vrtx_015B1npskreQgEjMrfsdjH1m","model":"claude-haiku-4.5"}
62+
63+
data: {"choices":[{"index":0,"delta":{"content":"\n- Status","role":"assistant"}}],"created":1769198062,"id":"msg_vrtx_015B1npskreQgEjMrfsdjH1m","model":"claude-haiku-4.5"}
64+
65+
data: {"choices":[{"index":0,"delta":{"content":": Up","role":"assistant"}}],"created":1769198062,"id":"msg_vrtx_015B1npskreQgEjMrfsdjH1m","model":"claude-haiku-4.5"}
66+
67+
data: {"choices":[{"index":0,"delta":{"content":" to date","role":"assistant"}}],"created":1769198062,"id":"msg_vrtx_015B1npskreQgEjMrfsdjH1m","model":"claude-haiku-4.5"}
68+
69+
data: {"choices":[{"finish_reason":"stop","index":0,"delta":{"content":null}}],"created":1769198062,"id":"msg_vrtx_015B1npskreQgEjMrfsdjH1m","usage":{"completion_tokens":85,"prompt_tokens":25989,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":26074},"model":"claude-haiku-4.5"}
70+
71+
data: [DONE]
72+
73+

‎fixtures/openai/chatcompletions/streaming_injected_tool_nonzero_index.txtar‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
Streaming response where the provider returns text content followed by an injected tool call at index 1 (instead of index 0).
2+
This can happen when the provider incorrectly continues indexing from a previous response.
3+
This tests that nil entries are removed from the tool calls array caused by non-zero starting indices.
4+
5+
-- request --
6+
{
7+
"messages": [
8+
{
9+
"content": "<current_datetime>2026-01-23T20:22:43.781Z</current_datetime>\n\nI want you to do to this in order:\n1) create a file in my current directory with name \"test.txt\"\n2) list all my coder workspaces",
10+
"role": "user"
11+
}
12+
],
13+
"model": "claude-haiku-4.5",
14+
"n": 1,
15+
"temperature": 1,
16+
"parallel_tool_calls": false,
17+
"stream_options": {
18+
"include_usage": true
19+
},
20+
"stream": true
21+
}
22+
23+
-- streaming --
24+
data: {"choices":[{"index":0,"delta":{"content":"Now","role":"assistant"}}],"created":1769199774,"id":"msg_vrtx_01Fiieb5Z3kqJf9a3FwvLkky","model":"claude-haiku-4.5"}
25+
26+
data: {"choices":[{"index":0,"delta":{"content":" listing","role":"assistant"}}],"created":1769199774,"id":"msg_vrtx_01Fiieb5Z3kqJf9a3FwvLkky","model":"claude-haiku-4.5"}
27+
28+
data: {"choices":[{"index":0,"delta":{"content":" your","role":"assistant"}}],"created":1769199774,"id":"msg_vrtx_01Fiieb5Z3kqJf9a3FwvLkky","model":"claude-haiku-4.5"}
29+
30+
data: {"choices":[{"index":0,"delta":{"content":" C","role":"assistant"}}],"created":1769199774,"id":"msg_vrtx_01Fiieb5Z3kqJf9a3FwvLkky","model":"claude-haiku-4.5"}
31+
32+
data: {"choices":[{"index":0,"delta":{"content":"oder workspaces:","role":"assistant"}}],"created":1769199774,"id":"msg_vrtx_01Fiieb5Z3kqJf9a3FwvLkky","model":"claude-haiku-4.5"}
33+
34+
data: {"choices":[{"index":0,"delta":{"content":null,"tool_calls":[{"function":{"name":"bmcp_coder_coder_list_workspaces"},"id":"toolu_vrtx_01DbFqUgk6aAtJ4nDBqzFWDF","index":1,"type":"function"}]}}],"created":1769199774,"id":"msg_vrtx_01Fiieb5Z3kqJf9a3FwvLkky","model":"claude-haiku-4.5"}
35+
36+
data: {"choices":[{"index":0,"delta":{"content":null,"tool_calls":[{"function":{"arguments":""},"index":1}]}}],"created":1769199774,"id":"msg_vrtx_01Fiieb5Z3kqJf9a3FwvLkky","model":"claude-haiku-4.5"}
37+
38+
data: {"choices":[{"finish_reason":"tool_calls","index":0,"delta":{"content":null}}],"created":1769199774,"id":"msg_vrtx_01Fiieb5Z3kqJf9a3FwvLkky","usage":{"completion_tokens":58,"prompt_tokens":25939,"prompt_tokens_details":{"cached_tokens":25429},"total_tokens":25997},"model":"claude-haiku-4.5"}
39+
40+
data: [DONE]
41+
42+
43+
-- streaming/tool-call --
44+
data: {"choices":[{"index":0,"delta":{"content":"Done","role":"assistant"}}],"created":1769199776,"id":"msg_vrtx_01RVxamMyw1DBtpoENDpmnQK","model":"claude-haiku-4.5"}
45+
46+
data: {"choices":[{"index":0,"delta":{"content":"! I create","role":"assistant"}}],"created":1769199776,"id":"msg_vrtx_01RVxamMyw1DBtpoENDpmnQK","model":"claude-haiku-4.5"}
47+
48+
data: {"choices":[{"index":0,"delta":{"content":"d `","role":"assistant"}}],"created":1769199776,"id":"msg_vrtx_01RVxamMyw1DBtpoENDpmnQK","model":"claude-haiku-4.5"}
49+
50+
data: {"choices":[{"index":0,"delta":{"content":"test.txt` in","role":"assistant"}}],"created":1769199776,"id":"msg_vrtx_01RVxamMyw1DBtpoENDpmnQK","model":"claude-haiku-4.5"}
51+
52+
data: {"choices":[{"index":0,"delta":{"content":" your current directory.","role":"assistant"}}],"created":1769199776,"id":"msg_vrtx_01RVxamMyw1DBtpoENDpmnQK","model":"claude-haiku-4.5"}
53+
54+
data: {"choices":[{"index":0,"delta":{"content":" You","role":"assistant"}}],"created":1769199776,"id":"msg_vrtx_01RVxamMyw1DBtpoENDpmnQK","model":"claude-haiku-4.5"}
55+
56+
data: {"choices":[{"index":0,"delta":{"content":" have","role":"assistant"}}],"created":1769199776,"id":"msg_vrtx_01RVxamMyw1DBtpoENDpmnQK","model":"claude-haiku-4.5"}
57+
58+
data: {"choices":[{"index":0,"delta":{"content":" 1","role":"assistant"}}],"created":1769199776,"id":"msg_vrtx_01RVxamMyw1DBtpoENDpmnQK","model":"claude-haiku-4.5"}
59+
60+
data: {"choices":[{"index":0,"delta":{"content":" ","role":"assistant"}}],"created":1769199776,"id":"msg_vrtx_01RVxamMyw1DBtpoENDpmnQK","model":"claude-haiku-4.5"}
61+
62+
data: {"choices":[{"index":0,"delta":{"content":"Coder workspace:\n\n-","role":"assistant"}}],"created":1769199776,"id":"msg_vrtx_01RVxamMyw1DBtpoENDpmnQK","model":"claude-haiku-4.5"}
63+
64+
data: {"choices":[{"index":0,"delta":{"content":" **test-scf** (docker","role":"assistant"}}],"created":1769199776,"id":"msg_vrtx_01RVxamMyw1DBtpoENDpmnQK","model":"claude-haiku-4.5"}
65+
66+
data: {"choices":[{"index":0,"delta":{"content":" template)","role":"assistant"}}],"created":1769199776,"id":"msg_vrtx_01RVxamMyw1DBtpoENDpmnQK","model":"claude-haiku-4.5"}
67+
68+
data: {"choices":[{"finish_reason":"stop","index":0,"delta":{"content":null}}],"created":1769199776,"id":"msg_vrtx_01RVxamMyw1DBtpoENDpmnQK","usage":{"completion_tokens":39,"prompt_tokens":26166,"prompt_tokens_details":{"cached_tokens":25934},"total_tokens":26205},"model":"claude-haiku-4.5"}
69+
70+
data: [DONE]
71+
72+

‎intercept/chatcompletions/streaming.go‎

Lines changed: 40 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"errors"
88
"fmt"
99
"net/http"
10+
"slices"
1011
"strings"
1112
"time"
1213

@@ -156,16 +157,28 @@ func (i *StreamingInterception) ProcessRequest(w http.ResponseWriter, r *http.Re
156157
}
157158
}
158159

159-
// Builtin tools are not intercepted.
160-
if toolCall != nil && i.getInjectedToolByName(toolCall.Name) == nil {
161-
_ = i.recorder.RecordToolUsage(streamCtx, &recorder.ToolUsageRecord{
162-
InterceptionID: i.ID().String(),
163-
MsgID: processor.getMsgID(),
164-
Tool: toolCall.Name,
165-
Args: i.unmarshalArgs(toolCall.Arguments),
166-
Injected: false,
167-
})
168-
toolCall = nil
160+
if toolCall != nil {
161+
// Builtin tools are not intercepted.
162+
if i.getInjectedToolByName(toolCall.Name) == nil {
163+
_ = i.recorder.RecordToolUsage(streamCtx, &recorder.ToolUsageRecord{
164+
InterceptionID: i.ID().String(),
165+
MsgID: processor.getMsgID(),
166+
Tool: toolCall.Name,
167+
Args: i.unmarshalArgs(toolCall.Arguments),
168+
Injected: false,
169+
})
170+
toolCall = nil
171+
} else {
172+
// When the provider responds with only tool calls (no text content),
173+
// no chunks are relayed to the client, so the stream is not yet
174+
// initiated. Initiate it here so the SSE headers are sent and the
175+
// ping ticker is started, preventing client timeout during tool invocation.
176+
// Only initiate if no stream error, if there's an error, we'll return
177+
// an HTTP error response instead of starting an SSE stream.
178+
if stream.Err() == nil {
179+
events.InitiateStream(w)
180+
}
181+
}
169182
}
170183

171184
if prompt != nil {
@@ -247,7 +260,13 @@ func (i *StreamingInterception) ProcessRequest(w http.ResponseWriter, r *http.Re
247260

248261
// Invoke the injected tool, and use the tool result to make a subsequent request to the upstream.
249262
// Append the completion from this stream as context.
250-
i.req.Messages = append(i.req.Messages, processor.getLastCompletion().ToParam())
263+
// Some providers may return tool calls with non-zero starting indices,
264+
// resulting in nil entries in the array that must be removed.
265+
completion := processor.getLastCompletion()
266+
if completion != nil {
267+
compactToolCalls(completion)
268+
i.req.Messages = append(i.req.Messages, completion.ToParam())
269+
}
251270

252271
id := toolCall.ID
253272
args := i.unmarshalArgs(toolCall.Arguments)
@@ -494,3 +513,13 @@ func (s *streamProcessor) getLastUsage() openai.CompletionUsage {
494513
func (s *streamProcessor) getCumulativeUsage() openai.CompletionUsage {
495514
return s.cumulativeUsage
496515
}
516+
517+
// compactToolCalls removes nil/empty tool call entries (without an ID).
518+
func compactToolCalls(msg *openai.ChatCompletionMessage) {
519+
if msg == nil || len(msg.ToolCalls) == 0 {
520+
return
521+
}
522+
msg.ToolCalls = slices.DeleteFunc(msg.ToolCalls, func(tc openai.ChatCompletionMessageToolCallUnion) bool {
523+
return tc.ID == ""
524+
})
525+
}

0 commit comments

Comments
 (0)