feat(autoparser): prefer chat deltas from backends when emitted

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

Author: mudler

core/backend/llm.go

@@ -36,6 +36,27 @@ type TokenUsage struct {
 	Completion             int
 	TimingPromptProcessing float64
 	TimingTokenGeneration  float64
+	ChatDeltas             []*proto.ChatDelta // per-chunk deltas from C++ autoparser (only set during streaming)
+}
+
+// HasChatDeltaContent returns true if any chat delta carries content or reasoning text.
+// Used to decide whether to prefer C++ autoparser deltas over Go-side tag extraction.
+func (t TokenUsage) HasChatDeltaContent() bool {
+	for _, d := range t.ChatDeltas {
+		if d.Content != "" || d.ReasoningContent != "" {
+			return true
+		}
+	}
+	return false
+}
+
+// ChatDeltaReasoningAndContent extracts accumulated reasoning and content from chat deltas.
+func (t TokenUsage) ChatDeltaReasoningAndContent() (reasoning, content string) {
+	for _, d := range t.ChatDeltas {
+		content += d.Content
+		reasoning += d.ReasoningContent
+	}
+	return reasoning, content
 }
 
 // ModelInferenceFunc is a test-friendly indirection to call model inference logic.
@@ -171,6 +192,9 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima
 					allChatDeltas = append(allChatDeltas, reply.ChatDeltas...)
 				}
 
+				// Attach per-chunk chat deltas to tokenUsage so the callback can use them
+				tokenUsage.ChatDeltas = reply.ChatDeltas
+
 				// Parse logprobs from reply if present (collect from last chunk that has them)
 				if len(reply.Logprobs) > 0 {
 					var parsedLogprobs schema.Logprobs
@@ -200,6 +224,9 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima
 				if len(msg) == 0 {
 					tokenCallback("", tokenUsage)
 				}
+
+				// Clear per-chunk deltas so they don't leak to the next chunk
+				tokenUsage.ChatDeltas = nil
 			})
 			if len(allChatDeltas) > 0 {
 				xlog.Debug("[ChatDeltas] streaming completed, accumulated deltas from C++ autoparser", "total_deltas", len(allChatDeltas))

core/http/endpoints/openai/chat.go

@@ -81,7 +81,17 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
 		extractor := reason.NewReasoningExtractor(thinkingStartToken, config.ReasoningConfig)
 
 		_, _, _, err := ComputeChoices(req, s, config, cl, startupOptions, loader, func(s string, c *[]schema.Choice) {}, func(s string, tokenUsage backend.TokenUsage) bool {
-			reasoningDelta, contentDelta := extractor.ProcessToken(s)
+			var reasoningDelta, contentDelta string
+
+			// Prefer pre-parsed chat deltas from C++ autoparser when available
+			if tokenUsage.HasChatDeltaContent() {
+				reasoningDelta, contentDelta = tokenUsage.ChatDeltaReasoningAndContent()
+				// Keep extractor state consistent for fallback
+				extractor.ProcessToken(s)
+			} else {
+				// Fallback: Go-side extraction from raw text
+				reasoningDelta, contentDelta = extractor.ProcessToken(s)
+			}
 
 			usage := schema.OpenAIUsage{
 				PromptTokens:     tokenUsage.Prompt,
@@ -133,7 +143,18 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
 
 		_, tokenUsage, chatDeltas, err := ComputeChoices(req, prompt, config, cl, startupOptions, loader, func(s string, c *[]schema.Choice) {}, func(s string, usage backend.TokenUsage) bool {
 			result += s
-			reasoningDelta, contentDelta := extractor.ProcessToken(s)
+
+			var reasoningDelta, contentDelta string
+
+			// Prefer pre-parsed chat deltas from C++ autoparser when available
+			if usage.HasChatDeltaContent() {
+				reasoningDelta, contentDelta = usage.ChatDeltaReasoningAndContent()
+				// Keep extractor state consistent for fallback
+				extractor.ProcessToken(s)
+			} else {
+				// Fallback: Go-side extraction from raw text
+				reasoningDelta, contentDelta = extractor.ProcessToken(s)
+			}
 
 			// Emit reasoning deltas in their own SSE chunks before any tool-call chunks
 			// (OpenAI spec: reasoning and tool_calls never share a delta)

core/http/endpoints/openresponses/responses.go

@@ -1819,7 +1819,14 @@ func handleOpenResponsesStream(c echo.Context, responseID string, createdAt int6
 
 				// If no tool calls detected yet, handle reasoning and text
 				if !inToolCallMode {
-					reasoningDelta, contentDelta := extractor.ProcessToken(token)
+					var reasoningDelta, contentDelta string
+					// Prefer pre-parsed chat deltas from C++ autoparser when available
+					if tokenUsage.HasChatDeltaContent() {
+						reasoningDelta, contentDelta = tokenUsage.ChatDeltaReasoningAndContent()
+						extractor.ProcessToken(token) // keep state consistent
+					} else {
+						reasoningDelta, contentDelta = extractor.ProcessToken(token)
+					}
 
 					// Handle reasoning item
 					if extractor.Reasoning() != "" {
@@ -2338,7 +2345,15 @@ func handleOpenResponsesStream(c echo.Context, responseID string, createdAt int6
 	// Stream text deltas with reasoning extraction
 	tokenCallback := func(token string, tokenUsage backend.TokenUsage) bool {
 		accumulatedText += token
-		reasoningDelta, contentDelta := extractor.ProcessToken(token)
+
+		var reasoningDelta, contentDelta string
+		// Prefer pre-parsed chat deltas from C++ autoparser when available
+		if tokenUsage.HasChatDeltaContent() {
+			reasoningDelta, contentDelta = tokenUsage.ChatDeltaReasoningAndContent()
+			extractor.ProcessToken(token) // keep state consistent
+		} else {
+			reasoningDelta, contentDelta = extractor.ProcessToken(token)
+		}
 
 		// Handle reasoning item
 		if extractor.Reasoning() != "" {