Skip to content

Commit c4df162

Browse files
mudlermudler
committed
chore: be consistent and apply fallback to all endpoint
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
1 parent bdbba5d commit c4df162

3 files changed

Lines changed: 227 additions & 3 deletions

File tree

‎core/http/endpoints/anthropic/messages.go‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,35 @@ func handleAnthropicNonStream(c echo.Context, id string, input *schema.Anthropic
306306
if textContent != "" {
307307
contentBlocks = append([]schema.AnthropicContentBlock{{Type: "text", Text: textContent}}, contentBlocks...)
308308
}
309+
} else if !shouldUseFn && cfg.FunctionsConfig.AutomaticToolParsingFallback && result != "" {
310+
// Automatic tool parsing fallback: no tools in request but model emitted tool call markup
311+
parsed := functions.ParseFunctionCall(result, cfg.FunctionsConfig)
312+
if len(parsed) > 0 {
313+
stopReason = "tool_use"
314+
stripped := functions.StripToolCallMarkup(result)
315+
if stripped != "" {
316+
contentBlocks = append(contentBlocks, schema.AnthropicContentBlock{Type: "text", Text: stripped})
317+
}
318+
for i, fc := range parsed {
319+
var inputArgs map[string]interface{}
320+
if err := json.Unmarshal([]byte(fc.Arguments), &inputArgs); err != nil {
321+
inputArgs = map[string]interface{}{"raw": fc.Arguments}
322+
}
323+
toolCallID := fc.ID
324+
if toolCallID == "" {
325+
toolCallID = fmt.Sprintf("toolu_%s_%d", id, i)
326+
}
327+
contentBlocks = append(contentBlocks, schema.AnthropicContentBlock{
328+
Type: "tool_use",
329+
ID: toolCallID,
330+
Name: fc.Name,
331+
Input: inputArgs,
332+
})
333+
}
334+
} else {
335+
stopReason = "end_turn"
336+
contentBlocks = []schema.AnthropicContentBlock{{Type: "text", Text: result}}
337+
}
309338
} else {
310339
stopReason = "end_turn"
311340
contentBlocks = []schema.AnthropicContentBlock{
@@ -522,6 +551,51 @@ func handleAnthropicStream(c echo.Context, id string, input *schema.AnthropicReq
522551
}
523552
}
524553

554+
// Automatic tool parsing fallback for streaming: when no tools were requested
555+
// but the model emitted tool call markup, parse and emit as tool_use blocks.
556+
if !shouldUseFn && cfg.FunctionsConfig.AutomaticToolParsingFallback && accumulatedContent != "" && toolCallsEmitted == 0 {
557+
parsed := functions.ParseFunctionCall(accumulatedContent, cfg.FunctionsConfig)
558+
if len(parsed) > 0 {
559+
// Close the text content block
560+
sendAnthropicSSE(c, schema.AnthropicStreamEvent{
561+
Type: "content_block_stop",
562+
Index: currentBlockIndex,
563+
})
564+
currentBlockIndex++
565+
inToolCall = true
566+
567+
for i, fc := range parsed {
568+
toolCallID := fc.ID
569+
if toolCallID == "" {
570+
toolCallID = fmt.Sprintf("toolu_%s_%d", id, i)
571+
}
572+
sendAnthropicSSE(c, schema.AnthropicStreamEvent{
573+
Type: "content_block_start",
574+
Index: currentBlockIndex,
575+
ContentBlock: &schema.AnthropicContentBlock{
576+
Type: "tool_use",
577+
ID: toolCallID,
578+
Name: fc.Name,
579+
},
580+
})
581+
sendAnthropicSSE(c, schema.AnthropicStreamEvent{
582+
Type: "content_block_delta",
583+
Index: currentBlockIndex,
584+
Delta: &schema.AnthropicStreamDelta{
585+
Type: "input_json_delta",
586+
PartialJSON: fc.Arguments,
587+
},
588+
})
589+
sendAnthropicSSE(c, schema.AnthropicStreamEvent{
590+
Type: "content_block_stop",
591+
Index: currentBlockIndex,
592+
})
593+
currentBlockIndex++
594+
toolCallsEmitted++
595+
}
596+
}
597+
}
598+
525599
// No MCP tools to execute, close stream
526600
if !inToolCall {
527601
sendAnthropicSSE(c, schema.AnthropicStreamEvent{

‎core/http/endpoints/openai/chat.go‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -751,8 +751,8 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
751751
collectedToolCalls = mergeToolCallDeltas(collectedToolCalls, ev.Choices[0].Delta.ToolCalls)
752752
}
753753
}
754-
// Collect content for MCP conversation history
755-
if hasMCPToolsStream && ev.Choices[0].Delta != nil && ev.Choices[0].Delta.Content != nil {
754+
// Collect content for MCP conversation history and automatic tool parsing fallback
755+
if (hasMCPToolsStream || config.FunctionsConfig.AutomaticToolParsingFallback) && ev.Choices[0].Delta != nil && ev.Choices[0].Delta.Content != nil {
756756
if s, ok := ev.Choices[0].Delta.Content.(string); ok {
757757
collectedContent += s
758758
} else if sp, ok := ev.Choices[0].Delta.Content.(*string); ok && sp != nil {
@@ -857,6 +857,43 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
857857
}
858858
}
859859

860+
// Automatic tool parsing fallback for streaming: when no tools were
861+
// requested but the model emitted tool call markup, parse and emit them.
862+
if !shouldUseFn && config.FunctionsConfig.AutomaticToolParsingFallback && collectedContent != "" && !toolsCalled {
863+
parsed := functions.ParseFunctionCall(collectedContent, config.FunctionsConfig)
864+
for i, fc := range parsed {
865+
toolCallID := fc.ID
866+
if toolCallID == "" {
867+
toolCallID = id
868+
}
869+
toolCallMsg := schema.OpenAIResponse{
870+
ID: id,
871+
Created: created,
872+
Model: input.Model,
873+
Choices: []schema.Choice{{
874+
Delta: &schema.Message{
875+
Role: "assistant",
876+
ToolCalls: []schema.ToolCall{{
877+
Index: i,
878+
ID: toolCallID,
879+
Type: "function",
880+
FunctionCall: schema.FunctionCall{
881+
Name: fc.Name,
882+
Arguments: fc.Arguments,
883+
},
884+
}},
885+
},
886+
Index: 0,
887+
}},
888+
Object: "chat.completion.chunk",
889+
}
890+
respData, _ := json.Marshal(toolCallMsg)
891+
fmt.Fprintf(c.Response().Writer, "data: %s\n\n", respData)
892+
c.Response().Flush()
893+
toolsCalled = true
894+
}
895+
}
896+
860897
// No MCP tools to execute, send final stop message
861898
finishReason := FinishReasonStop
862899
if toolsCalled && len(input.Tools) > 0 {

‎core/http/endpoints/openresponses/responses.go‎

Lines changed: 114 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1013,6 +1013,35 @@ func handleBackgroundNonStream(ctx context.Context, store *ResponseStore, respon
10131013
Content: []schema.ORContentPart{makeOutputTextPartWithLogprobs(result, resultLogprobs)},
10141014
})
10151015
}
1016+
} else if !shouldUseFn && cfg.FunctionsConfig.AutomaticToolParsingFallback && result != "" {
1017+
// Automatic tool parsing fallback: no tools in request but model emitted tool call markup
1018+
parsed := functions.ParseFunctionCall(result, cfg.FunctionsConfig)
1019+
if len(parsed) > 0 {
1020+
stripped := functions.StripToolCallMarkup(result)
1021+
if stripped != "" {
1022+
allOutputItems = append(allOutputItems, schema.ORItemField{
1023+
Type: "message", ID: fmt.Sprintf("msg_%s", uuid.New().String()),
1024+
Status: "completed", Role: "assistant",
1025+
Content: []schema.ORContentPart{makeOutputTextPartWithLogprobs(stripped, resultLogprobs)},
1026+
})
1027+
}
1028+
for _, fc := range parsed {
1029+
toolCallID := fc.ID
1030+
if toolCallID == "" {
1031+
toolCallID = fmt.Sprintf("fc_%s", uuid.New().String())
1032+
}
1033+
allOutputItems = append(allOutputItems, schema.ORItemField{
1034+
Type: "function_call", ID: fmt.Sprintf("fc_%s", uuid.New().String()),
1035+
Status: "completed", CallID: toolCallID, Name: fc.Name, Arguments: fc.Arguments,
1036+
})
1037+
}
1038+
} else {
1039+
allOutputItems = append(allOutputItems, schema.ORItemField{
1040+
Type: "message", ID: fmt.Sprintf("msg_%s", uuid.New().String()),
1041+
Status: "completed", Role: "assistant",
1042+
Content: []schema.ORContentPart{makeOutputTextPartWithLogprobs(result, resultLogprobs)},
1043+
})
1044+
}
10161045
} else {
10171046
allOutputItems = append(allOutputItems, schema.ORItemField{
10181047
Type: "message", ID: fmt.Sprintf("msg_%s", uuid.New().String()),
@@ -1539,6 +1568,43 @@ func handleOpenResponsesNonStream(c echo.Context, responseID string, createdAt i
15391568
Content: []schema.ORContentPart{makeOutputTextPartWithLogprobs(cleanedResult, resultLogprobs)},
15401569
})
15411570
}
1571+
} else if !shouldUseFn && cfg.FunctionsConfig.AutomaticToolParsingFallback && cleanedResult != "" {
1572+
// Automatic tool parsing fallback: no tools in request but model emitted tool call markup
1573+
parsed := functions.ParseFunctionCall(cleanedResult, cfg.FunctionsConfig)
1574+
if len(parsed) > 0 {
1575+
stripped := functions.StripToolCallMarkup(cleanedResult)
1576+
if stripped != "" {
1577+
outputItems = append(outputItems, schema.ORItemField{
1578+
Type: "message",
1579+
ID: fmt.Sprintf("msg_%s", uuid.New().String()),
1580+
Status: "completed",
1581+
Role: "assistant",
1582+
Content: []schema.ORContentPart{makeOutputTextPartWithLogprobs(stripped, resultLogprobs)},
1583+
})
1584+
}
1585+
for _, fc := range parsed {
1586+
toolCallID := fc.ID
1587+
if toolCallID == "" {
1588+
toolCallID = fmt.Sprintf("fc_%s", uuid.New().String())
1589+
}
1590+
outputItems = append(outputItems, schema.ORItemField{
1591+
Type: "function_call",
1592+
ID: fmt.Sprintf("fc_%s", uuid.New().String()),
1593+
Status: "completed",
1594+
CallID: toolCallID,
1595+
Name: fc.Name,
1596+
Arguments: fc.Arguments,
1597+
})
1598+
}
1599+
} else {
1600+
outputItems = append(outputItems, schema.ORItemField{
1601+
Type: "message",
1602+
ID: fmt.Sprintf("msg_%s", uuid.New().String()),
1603+
Status: "completed",
1604+
Role: "assistant",
1605+
Content: []schema.ORContentPart{makeOutputTextPartWithLogprobs(cleanedResult, resultLogprobs)},
1606+
})
1607+
}
15421608
} else {
15431609
// Simple text response (include logprobs if available)
15441610
messageItem := schema.ORItemField{
@@ -2514,6 +2580,15 @@ func handleOpenResponsesStream(c echo.Context, responseID string, createdAt int6
25142580

25152581
result = finalCleanedResult
25162582

2583+
// Automatic tool parsing fallback for streaming: parse tool calls from accumulated text
2584+
var streamFallbackToolCalls []functions.FuncCallResults
2585+
if cfg.FunctionsConfig.AutomaticToolParsingFallback && result != "" {
2586+
streamFallbackToolCalls = functions.ParseFunctionCall(result, cfg.FunctionsConfig)
2587+
if len(streamFallbackToolCalls) > 0 {
2588+
result = functions.StripToolCallMarkup(result)
2589+
}
2590+
}
2591+
25172592
// Convert logprobs for streaming events
25182593
mcpStreamLogprobs := convertLogprobsForStreaming(noToolLogprobs)
25192594

@@ -2552,10 +2627,42 @@ func handleOpenResponsesStream(c echo.Context, responseID string, createdAt int6
25522627
})
25532628
sequenceNumber++
25542629

2630+
// Emit function_call items from automatic tool parsing fallback
2631+
for _, fc := range streamFallbackToolCalls {
2632+
toolCallID := fc.ID
2633+
if toolCallID == "" {
2634+
toolCallID = fmt.Sprintf("fc_%s", uuid.New().String())
2635+
}
2636+
outputIndex++
2637+
functionCallItem := &schema.ORItemField{
2638+
Type: "function_call",
2639+
ID: toolCallID,
2640+
Status: "completed",
2641+
CallID: toolCallID,
2642+
Name: fc.Name,
2643+
Arguments: fc.Arguments,
2644+
}
2645+
sendSSEEvent(c, &schema.ORStreamEvent{
2646+
Type: "response.output_item.added",
2647+
SequenceNumber: sequenceNumber,
2648+
OutputIndex: &outputIndex,
2649+
Item: functionCallItem,
2650+
})
2651+
sequenceNumber++
2652+
sendSSEEvent(c, &schema.ORStreamEvent{
2653+
Type: "response.output_item.done",
2654+
SequenceNumber: sequenceNumber,
2655+
OutputIndex: &outputIndex,
2656+
Item: functionCallItem,
2657+
})
2658+
sequenceNumber++
2659+
collectedOutputItems = append(collectedOutputItems, *functionCallItem)
2660+
}
2661+
25552662
// Emit response.completed
25562663
now := time.Now().Unix()
25572664

2558-
// Collect final output items (reasoning first, then message)
2665+
// Collect final output items (reasoning first, then messages, then tool calls)
25592666
var finalOutputItems []schema.ORItemField
25602667
// Add reasoning item if it exists
25612668
if currentReasoningID != "" && finalReasoning != "" {
@@ -2577,6 +2684,12 @@ func handleOpenResponsesStream(c echo.Context, responseID string, createdAt int6
25772684
} else {
25782685
finalOutputItems = append(finalOutputItems, *messageItem)
25792686
}
2687+
// Add function_call items from fallback
2688+
for _, item := range collectedOutputItems {
2689+
if item.Type == "function_call" {
2690+
finalOutputItems = append(finalOutputItems, item)
2691+
}
2692+
}
25802693
responseCompleted := buildORResponse(responseID, createdAt, &now, "completed", input, finalOutputItems, &schema.ORUsage{
25812694
InputTokens: noToolTokenUsage.Prompt,
25822695
OutputTokens: noToolTokenUsage.Completion,

0 commit comments

Comments
 (0)