chore: refactor endpoints to use same inferencing path, add automatic retrial mechanism in case of errors (#9029)

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

Author: mudler

core/backend/llm.go

@@ -38,6 +38,10 @@ type TokenUsage struct {
 	TimingTokenGeneration  float64
 }
 
+// ModelInferenceFunc is a test-friendly indirection to call model inference logic.
+// Tests can override this variable to provide a stub implementation.
+var ModelInferenceFunc = ModelInference
+
 func ModelInference(ctx context.Context, s string, messages schema.Messages, images, videos, audios []string, loader *model.ModelLoader, c *config.ModelConfig, cl *config.ModelConfigLoader, o *config.ApplicationConfig, tokenCallback func(string, TokenUsage) bool, tools string, toolChoice string, logprobs *int, topLogprobs *int, logitBias map[string]float64, metadata map[string]string) (func() (LLMResponse, error), error) {
 	modelFile := c.Model
 

core/http/endpoints/anthropic/messages.go

@@ -10,6 +10,7 @@ import (
 	"github.com/mudler/LocalAI/core/backend"
 	"github.com/mudler/LocalAI/core/config"
 	mcpTools "github.com/mudler/LocalAI/core/http/endpoints/mcp"
+	openaiEndpoint "github.com/mudler/LocalAI/core/http/endpoints/openai"
 	"github.com/mudler/LocalAI/core/http/middleware"
 	"github.com/mudler/LocalAI/core/schema"
 	"github.com/mudler/LocalAI/core/templates"
@@ -197,54 +198,24 @@ func handleAnthropicNonStream(c echo.Context, id string, input *schema.Anthropic
 			xlog.Debug("Anthropic MCP re-templating", "iteration", mcpIteration, "prompt_len", len(predInput))
 		}
 
-		images := []string{}
-		for _, m := range openAIReq.Messages {
-			images = append(images, m.StringImages...)
-		}
+		// Populate openAIReq fields for ComputeChoices
+		openAIReq.Tools = convertFuncsToOpenAITools(funcs)
+		openAIReq.ToolsChoice = input.ToolChoice
+		openAIReq.Metadata = input.Metadata
 
-		toolsJSON := ""
-		if len(funcs) > 0 {
-			openAITools := make([]functions.Tool, len(funcs))
-			for i, f := range funcs {
-				openAITools[i] = functions.Tool{Type: "function", Function: f}
-			}
-			if toolsBytes, err := json.Marshal(openAITools); err == nil {
-				toolsJSON = string(toolsBytes)
-			}
-		}
-		toolChoiceJSON := ""
-		if input.ToolChoice != nil {
-			if toolChoiceBytes, err := json.Marshal(input.ToolChoice); err == nil {
-				toolChoiceJSON = string(toolChoiceBytes)
-			}
+		var result string
+		cb := func(s string, c *[]schema.Choice) {
+			result = s
 		}
-
-		predFunc, err := backend.ModelInference(
-			input.Context, predInput, openAIReq.Messages, images, nil, nil, ml, cfg, cl, appConfig, nil, toolsJSON, toolChoiceJSON, nil, nil, nil, input.Metadata)
+		_, tokenUsage, chatDeltas, err := openaiEndpoint.ComputeChoices(openAIReq, predInput, cfg, cl, appConfig, ml, cb, nil)
 		if err != nil {
 			xlog.Error("Anthropic model inference failed", "error", err)
 			return sendAnthropicError(c, 500, "api_error", fmt.Sprintf("model inference failed: %v", err))
 		}
 
-		const maxEmptyRetries = 5
-		var prediction backend.LLMResponse
-		var result string
-		for attempt := 0; attempt <= maxEmptyRetries; attempt++ {
-			prediction, err = predFunc()
-			if err != nil {
-				xlog.Error("Anthropic prediction failed", "error", err)
-				return sendAnthropicError(c, 500, "api_error", fmt.Sprintf("prediction failed: %v", err))
-			}
-			result = backend.Finetune(*cfg, predInput, prediction.Response)
-			if result != "" || !shouldUseFn {
-				break
-			}
-			xlog.Warn("Anthropic: retrying prediction due to empty backend response", "attempt", attempt+1, "maxRetries", maxEmptyRetries)
-		}
-
 		// Try pre-parsed tool calls from C++ autoparser first, fall back to text parsing
 		var toolCalls []functions.FuncCallResults
-		if deltaToolCalls := functions.ToolCallsFromChatDeltas(prediction.ChatDeltas); len(deltaToolCalls) > 0 {
+		if deltaToolCalls := functions.ToolCallsFromChatDeltas(chatDeltas); len(deltaToolCalls) > 0 {
 			xlog.Debug("[ChatDeltas] Anthropic: using pre-parsed tool calls", "count", len(deltaToolCalls))
 			toolCalls = deltaToolCalls
 		} else {
@@ -350,8 +321,8 @@ func handleAnthropicNonStream(c echo.Context, id string, input *schema.Anthropic
 			StopReason: &stopReason,
 			Content:    contentBlocks,
 			Usage: schema.AnthropicUsage{
-				InputTokens:  prediction.Usage.Prompt,
-				OutputTokens: prediction.Usage.Completion,
+				InputTokens:  tokenUsage.Prompt,
+				OutputTokens: tokenUsage.Completion,
 			},
 		}
 
@@ -397,12 +368,6 @@ func handleAnthropicStream(c echo.Context, id string, input *schema.AnthropicReq
 			xlog.Debug("Anthropic MCP stream re-templating", "iteration", mcpIteration)
 		}
 
-		openAIMessages := openAIReq.Messages
-		images := []string{}
-		for _, m := range openAIMessages {
-			images = append(images, m.StringImages...)
-		}
-
 		// Track accumulated content for tool call detection
 		accumulatedContent := ""
 		currentBlockIndex := 0
@@ -481,38 +446,19 @@ func handleAnthropicStream(c echo.Context, id string, input *schema.AnthropicReq
 			return true
 		}
 
-		toolsJSON := ""
-		if len(funcs) > 0 {
-			openAITools := make([]functions.Tool, len(funcs))
-			for i, f := range funcs {
-				openAITools[i] = functions.Tool{Type: "function", Function: f}
-			}
-			if toolsBytes, err := json.Marshal(openAITools); err == nil {
-				toolsJSON = string(toolsBytes)
-			}
-		}
-		toolChoiceJSON := ""
-		if input.ToolChoice != nil {
-			if toolChoiceBytes, err := json.Marshal(input.ToolChoice); err == nil {
-				toolChoiceJSON = string(toolChoiceBytes)
-			}
-		}
+		// Populate openAIReq fields for ComputeChoices
+		openAIReq.Tools = convertFuncsToOpenAITools(funcs)
+		openAIReq.ToolsChoice = input.ToolChoice
+		openAIReq.Metadata = input.Metadata
 
-		predFunc, err := backend.ModelInference(
-			input.Context, predInput, openAIMessages, images, nil, nil, ml, cfg, cl, appConfig, tokenCallback, toolsJSON, toolChoiceJSON, nil, nil, nil, input.Metadata)
+		_, tokenUsage, chatDeltas, err := openaiEndpoint.ComputeChoices(openAIReq, predInput, cfg, cl, appConfig, ml, func(s string, c *[]schema.Choice) {}, tokenCallback)
 		if err != nil {
 			xlog.Error("Anthropic stream model inference failed", "error", err)
 			return sendAnthropicError(c, 500, "api_error", fmt.Sprintf("model inference failed: %v", err))
 		}
 
-		prediction, err := predFunc()
-		if err != nil {
-			xlog.Error("Anthropic stream prediction failed", "error", err)
-			return sendAnthropicError(c, 500, "api_error", fmt.Sprintf("prediction failed: %v", err))
-		}
-
 		// Also check chat deltas for tool calls
-		if deltaToolCalls := functions.ToolCallsFromChatDeltas(prediction.ChatDeltas); len(deltaToolCalls) > 0 && len(collectedToolCalls) == 0 {
+		if deltaToolCalls := functions.ToolCallsFromChatDeltas(chatDeltas); len(deltaToolCalls) > 0 && len(collectedToolCalls) == 0 {
 			collectedToolCalls = deltaToolCalls
 		}
 
@@ -595,7 +541,7 @@ func handleAnthropicStream(c echo.Context, id string, input *schema.AnthropicReq
 				StopReason: &stopReason,
 			},
 			Usage: &schema.AnthropicUsage{
-				OutputTokens: prediction.Usage.Completion,
+				OutputTokens: tokenUsage.Completion,
 			},
 		})
 
@@ -613,6 +559,14 @@ func handleAnthropicStream(c echo.Context, id string, input *schema.AnthropicReq
 	return nil
 }
 
+func convertFuncsToOpenAITools(funcs functions.Functions) []functions.Tool {
+	tools := make([]functions.Tool, len(funcs))
+	for i, f := range funcs {
+		tools[i] = functions.Tool{Type: "function", Function: f}
+	}
+	return tools
+}
+
 func sendAnthropicSSE(c echo.Context, event schema.AnthropicStreamEvent) {
 	data, err := json.Marshal(event)
 	if err != nil {