Skip to content

chore: refactor endpoints to use same inferencing path, add automatic retrial mechanism in case of errors #9029

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mudler merged 2 commits into from
Mar 16, 2026
Merged

chore: refactor endpoints to use same inferencing path, add automatic retrial mechanism in case of errors #9029

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9553243
chore: refactor endpoints to use same inferencing path, add automatic…
mudler Mar 16, 2026
92494c5
Merge branch 'master' into chore/refactor-retry
mudler Mar 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions core/backend/llm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ type TokenUsage struct {
TimingTokenGeneration float64
}

// ModelInferenceFunc is a test-friendly indirection to call model inference logic.
// Tests can override this variable to provide a stub implementation.
var ModelInferenceFunc = ModelInference

func ModelInference(ctx context.Context, s string, messages schema.Messages, images, videos, audios []string, loader *model.ModelLoader, c *config.ModelConfig, cl *config.ModelConfigLoader, o *config.ApplicationConfig, tokenCallback func(string, TokenUsage) bool, tools string, toolChoice string, logprobs *int, topLogprobs *int, logitBias map[string]float64, metadata map[string]string) (func() (LLMResponse, error), error) {
modelFile := c.Model

Expand Down
100 changes: 27 additions & 73 deletions core/http/endpoints/anthropic/messages.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/mudler/LocalAI/core/backend"
"github.com/mudler/LocalAI/core/config"
mcpTools "github.com/mudler/LocalAI/core/http/endpoints/mcp"
openaiEndpoint "github.com/mudler/LocalAI/core/http/endpoints/openai"
"github.com/mudler/LocalAI/core/http/middleware"
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/core/templates"
Expand Down Expand Up @@ -197,54 +198,24 @@ func handleAnthropicNonStream(c echo.Context, id string, input *schema.Anthropic
xlog.Debug("Anthropic MCP re-templating", "iteration", mcpIteration, "prompt_len", len(predInput))
}

images := []string{}
for _, m := range openAIReq.Messages {
images = append(images, m.StringImages...)
}
// Populate openAIReq fields for ComputeChoices
openAIReq.Tools = convertFuncsToOpenAITools(funcs)
openAIReq.ToolsChoice = input.ToolChoice
openAIReq.Metadata = input.Metadata

toolsJSON := ""
if len(funcs) > 0 {
openAITools := make([]functions.Tool, len(funcs))
for i, f := range funcs {
openAITools[i] = functions.Tool{Type: "function", Function: f}
}
if toolsBytes, err := json.Marshal(openAITools); err == nil {
toolsJSON = string(toolsBytes)
}
}
toolChoiceJSON := ""
if input.ToolChoice != nil {
if toolChoiceBytes, err := json.Marshal(input.ToolChoice); err == nil {
toolChoiceJSON = string(toolChoiceBytes)
}
var result string
cb := func(s string, c *[]schema.Choice) {
result = s
}

predFunc, err := backend.ModelInference(
input.Context, predInput, openAIReq.Messages, images, nil, nil, ml, cfg, cl, appConfig, nil, toolsJSON, toolChoiceJSON, nil, nil, nil, input.Metadata)
_, tokenUsage, chatDeltas, err := openaiEndpoint.ComputeChoices(openAIReq, predInput, cfg, cl, appConfig, ml, cb, nil)
if err != nil {
xlog.Error("Anthropic model inference failed", "error", err)
return sendAnthropicError(c, 500, "api_error", fmt.Sprintf("model inference failed: %v", err))
}

const maxEmptyRetries = 5
var prediction backend.LLMResponse
var result string
for attempt := 0; attempt <= maxEmptyRetries; attempt++ {
prediction, err = predFunc()
if err != nil {
xlog.Error("Anthropic prediction failed", "error", err)
return sendAnthropicError(c, 500, "api_error", fmt.Sprintf("prediction failed: %v", err))
}
result = backend.Finetune(*cfg, predInput, prediction.Response)
if result != "" || !shouldUseFn {
break
}
xlog.Warn("Anthropic: retrying prediction due to empty backend response", "attempt", attempt+1, "maxRetries", maxEmptyRetries)
}

// Try pre-parsed tool calls from C++ autoparser first, fall back to text parsing
var toolCalls []functions.FuncCallResults
if deltaToolCalls := functions.ToolCallsFromChatDeltas(prediction.ChatDeltas); len(deltaToolCalls) > 0 {
if deltaToolCalls := functions.ToolCallsFromChatDeltas(chatDeltas); len(deltaToolCalls) > 0 {
xlog.Debug("[ChatDeltas] Anthropic: using pre-parsed tool calls", "count", len(deltaToolCalls))
toolCalls = deltaToolCalls
} else {
Expand Down Expand Up @@ -350,8 +321,8 @@ func handleAnthropicNonStream(c echo.Context, id string, input *schema.Anthropic
StopReason: &stopReason,
Content: contentBlocks,
Usage: schema.AnthropicUsage{
InputTokens: prediction.Usage.Prompt,
OutputTokens: prediction.Usage.Completion,
InputTokens: tokenUsage.Prompt,
OutputTokens: tokenUsage.Completion,
},
}

Expand Down Expand Up @@ -397,12 +368,6 @@ func handleAnthropicStream(c echo.Context, id string, input *schema.AnthropicReq
xlog.Debug("Anthropic MCP stream re-templating", "iteration", mcpIteration)
}

openAIMessages := openAIReq.Messages
images := []string{}
for _, m := range openAIMessages {
images = append(images, m.StringImages...)
}

// Track accumulated content for tool call detection
accumulatedContent := ""
currentBlockIndex := 0
Expand Down Expand Up @@ -481,38 +446,19 @@ func handleAnthropicStream(c echo.Context, id string, input *schema.AnthropicReq
return true
}

toolsJSON := ""
if len(funcs) > 0 {
openAITools := make([]functions.Tool, len(funcs))
for i, f := range funcs {
openAITools[i] = functions.Tool{Type: "function", Function: f}
}
if toolsBytes, err := json.Marshal(openAITools); err == nil {
toolsJSON = string(toolsBytes)
}
}
toolChoiceJSON := ""
if input.ToolChoice != nil {
if toolChoiceBytes, err := json.Marshal(input.ToolChoice); err == nil {
toolChoiceJSON = string(toolChoiceBytes)
}
}
// Populate openAIReq fields for ComputeChoices
openAIReq.Tools = convertFuncsToOpenAITools(funcs)
openAIReq.ToolsChoice = input.ToolChoice
openAIReq.Metadata = input.Metadata

predFunc, err := backend.ModelInference(
input.Context, predInput, openAIMessages, images, nil, nil, ml, cfg, cl, appConfig, tokenCallback, toolsJSON, toolChoiceJSON, nil, nil, nil, input.Metadata)
_, tokenUsage, chatDeltas, err := openaiEndpoint.ComputeChoices(openAIReq, predInput, cfg, cl, appConfig, ml, func(s string, c *[]schema.Choice) {}, tokenCallback)
if err != nil {
xlog.Error("Anthropic stream model inference failed", "error", err)
return sendAnthropicError(c, 500, "api_error", fmt.Sprintf("model inference failed: %v", err))
}

prediction, err := predFunc()
if err != nil {
xlog.Error("Anthropic stream prediction failed", "error", err)
return sendAnthropicError(c, 500, "api_error", fmt.Sprintf("prediction failed: %v", err))
}

// Also check chat deltas for tool calls
if deltaToolCalls := functions.ToolCallsFromChatDeltas(prediction.ChatDeltas); len(deltaToolCalls) > 0 && len(collectedToolCalls) == 0 {
if deltaToolCalls := functions.ToolCallsFromChatDeltas(chatDeltas); len(deltaToolCalls) > 0 && len(collectedToolCalls) == 0 {
collectedToolCalls = deltaToolCalls
}

Expand Down Expand Up @@ -595,7 +541,7 @@ func handleAnthropicStream(c echo.Context, id string, input *schema.AnthropicReq
StopReason: &stopReason,
},
Usage: &schema.AnthropicUsage{
OutputTokens: prediction.Usage.Completion,
OutputTokens: tokenUsage.Completion,
},
})

Expand All @@ -613,6 +559,14 @@ func handleAnthropicStream(c echo.Context, id string, input *schema.AnthropicReq
return nil
}

func convertFuncsToOpenAITools(funcs functions.Functions) []functions.Tool {
tools := make([]functions.Tool, len(funcs))
for i, f := range funcs {
tools[i] = functions.Tool{Type: "function", Function: f}
}
return tools
}

func sendAnthropicSSE(c echo.Context, event schema.AnthropicStreamEvent) {
data, err := json.Marshal(event)
if err != nil {
Expand Down
Loading
Loading