fix(streaming): skip chat deltas for role-init elements to prevent first token duplication

When TASK_RESPONSE_TYPE_OAI_CHAT is used, the first streaming token
produces a JSON array with two elements: a role-init chunk and the
actual content chunk. The grpc-server loop called attach_chat_deltas
for both elements with the same raw_result pointer, stamping the first
token's ChatDelta.Content on both replies. The Go side accumulated both,
emitting the first content token twice to SSE clients.

Fix: in the array iteration loops in PredictStream, detect role-init
elements (delta has "role" key) and skip attach_chat_deltas for them.
Only content/reasoning elements get chat deltas attached.

Reasoning models are unaffected because their first token goes into
reasoning_content, not content.

Author: mudler

backend/cpp/llama-cpp/grpc-server.cpp

@@ -1716,12 +1716,23 @@ class BackendServiceImpl final : public backend::Backend::Service {
             }
         };
 
-        // Process first result
+        // Process first result.
+        // When TASK_RESPONSE_TYPE_OAI_CHAT is used, the first token may
+        // produce a JSON array with a role-init element followed by the
+        // actual content element. We must only attach chat deltas to the
+        // content element — attaching to both would duplicate the first
+        // token since oaicompat_msg_diffs is the same for both.
         json first_res_json = first_result->to_json();
         if (first_res_json.is_array()) {
             for (const auto & res : first_res_json) {
                 auto reply = build_reply_from_json(res, first_result.get());
-                attach_chat_deltas(reply, first_result.get());
+                // Skip chat deltas for role-init elements (have "role" in
+                // delta but no content/reasoning diffs of their own).
+                bool is_role_init = res.contains("choices") && !res["choices"].empty() &&
+                                    res["choices"][0].value("delta", json::object()).contains("role");
+                if (!is_role_init) {
+                    attach_chat_deltas(reply, first_result.get());
+                }
                 writer->Write(reply);
             }
         } else {
@@ -1745,7 +1756,11 @@ class BackendServiceImpl final : public backend::Backend::Service {
             if (res_json.is_array()) {
                 for (const auto & res : res_json) {
                     auto reply = build_reply_from_json(res, result.get());
-                    attach_chat_deltas(reply, result.get());
+                    bool is_role_init = res.contains("choices") && !res["choices"].empty() &&
+                                        res["choices"][0].value("delta", json::object()).contains("role");
+                    if (!is_role_init) {
+                        attach_chat_deltas(reply, result.get());
+                    }
                     writer->Write(reply);
                 }
             } else {

tests/e2e/mock-backend/main.go

@@ -219,6 +219,33 @@ func (m *MockBackend) PredictStream(in *pb.PredictOptions, stream pb.Backend_Pre
 		return nil
 	}
 
+	// Simulate the first-token streaming for non-reasoning models.
+	// The C++ backend with TASK_RESPONSE_TYPE_OAI_CHAT sends a role-init
+	// reply (no ChatDeltas) followed by content replies (with ChatDeltas).
+	// Previously, attach_chat_deltas was called for both, duplicating the
+	// first token. After the fix, role-init replies have no ChatDeltas.
+	if strings.Contains(in.Prompt, "FIRST_TOKEN_DUP") {
+		content := "Hello! How can I help you today?"
+		// Reply 1: role-init (no ChatDeltas — fix applied)
+		if err := stream.Send(&pb.Reply{
+			Message: []byte{},
+		}); err != nil {
+			return err
+		}
+		// Reply 2+: content tokens with ChatDeltas
+		for _, word := range strings.Split(content, "") {
+			if err := stream.Send(&pb.Reply{
+				Message: []byte(word),
+				ChatDeltas: []*pb.ChatDelta{
+					{Content: word},
+				},
+			}); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
 	var toStream string
 	toolName := mockToolNameFromRequest(in)
 	if toolName != "" && !promptHasToolResults(in.Prompt) {

tests/e2e/mock_backend_test.go

@@ -509,4 +509,65 @@ var _ = Describe("Mock Backend E2E Tests", Label("MockBackend"), func() {
 			})
 		})
 	})
+
+	// Regression test: first content token duplicated in streaming for
+	// non-reasoning models. The C++ backend's PredictStream with
+	// TASK_RESPONSE_TYPE_OAI_CHAT returns a two-element JSON array for the
+	// first token (role-init + content), and attach_chat_deltas stamps
+	// both with the same ChatDelta.Content, causing the Go side to emit
+	// the first token twice.
+	Describe("First Token Duplication", Label("FirstTokenDup"), func() {
+		It("should not duplicate the first content token in streaming", func() {
+			body := `{
+				"model": "mock-model-autoparser",
+				"messages": [{"role": "user", "content": "FIRST_TOKEN_DUP"}],
+				"stream": true
+			}`
+			req, err := http.NewRequest("POST", apiURL+"/chat/completions", strings.NewReader(body))
+			Expect(err).ToNot(HaveOccurred())
+			req.Header.Set("Content-Type", "application/json")
+
+			httpClient := &http.Client{Timeout: 60 * time.Second}
+			resp, err := httpClient.Do(req)
+			Expect(err).ToNot(HaveOccurred())
+			defer resp.Body.Close()
+			Expect(resp.StatusCode).To(Equal(200))
+
+			data, err := io.ReadAll(resp.Body)
+			Expect(err).ToNot(HaveOccurred())
+
+			// Collect all content deltas
+			var contentParts []string
+			for _, line := range strings.Split(string(data), "\n") {
+				line = strings.TrimSpace(line)
+				if !strings.HasPrefix(line, "data: ") || line == "data: [DONE]" {
+					continue
+				}
+				var chunk map[string]any
+				if err := json.Unmarshal([]byte(strings.TrimPrefix(line, "data: ")), &chunk); err != nil {
+					continue
+				}
+				choices, _ := chunk["choices"].([]any)
+				if len(choices) == 0 {
+					continue
+				}
+				delta, _ := choices[0].(map[string]any)["delta"].(map[string]any)
+				if delta == nil {
+					continue
+				}
+				if content, ok := delta["content"].(string); ok && content != "" {
+					contentParts = append(contentParts, content)
+				}
+			}
+
+			fullContent := strings.Join(contentParts, "")
+
+			// The first token "Hello" should appear exactly once at the start
+			Expect(strings.Count(fullContent, "Hello")).To(Equal(1),
+				"First token 'Hello' was duplicated. Full streamed content: %q "+
+					"(content parts: %v)", fullContent, contentParts)
+			Expect(fullContent).To(HavePrefix("Hello"),
+				"Content should start with 'Hello', got: %q", fullContent)
+		})
+	})
 })