Fix streaming reasoning conversion to preserve extra fields (#11669)

Co-authored-by: David S. Batista <dsbatista@gmail.com>

Author: shr8769

haystack/components/generators/chat/openai_responses.py

@@ -629,7 +629,7 @@ def _convert_response_to_chat_message(responses: Response | ParsedResponse) -> C
     )
 
 
-def _convert_response_chunk_to_streaming_chunk(
+def _convert_response_chunk_to_streaming_chunk(  # noqa: PLR0911
     chunk: ResponseStreamEvent, previous_chunks: list[StreamingChunk], component_info: ComponentInfo | None = None
 ) -> StreamingChunk:
     """
@@ -669,6 +669,21 @@ def _convert_response_chunk_to_streaming_chunk(
                 meta={"received_at": datetime.now().isoformat()},
             )
 
+    elif chunk.type == "response.output_item.done":
+        # The done event carries the completed reasoning item, which includes encrypted_content
+        # when include=["reasoning.encrypted_content"] was requested. Without this handler the
+        # event falls through to the generic default and reasoning=None, so encrypted_content
+        # is never available for multi-turn conversations.
+        if chunk.item.type == "reasoning":
+            reasoning = ReasoningContent(reasoning_text="", extra=chunk.item.to_dict())
+            return StreamingChunk(
+                content="",
+                component_info=component_info,
+                index=chunk.output_index,
+                reasoning=reasoning,
+                meta={"received_at": datetime.now().isoformat()},
+            )
+
     elif chunk.type == "response.completed":
         # This means a full response is finished
         # If there are tool_calls present in the final output we mark finish_reason as tool_calls otherwise it's
@@ -812,7 +827,16 @@ def _convert_streaming_chunks_to_chat_message(chunks: list[StreamingChunk]) -> C
     # function calls without reasoning ids are not supported by the API
     reasoning = None
     if reasoning_id:
-        reasoning = ReasoningContent(reasoning_text=reasoning_text, extra={"id": reasoning_id, "type": "reasoning"})
+        # Preserve all extra fields from streaming chunks (e.g. encrypted_content) while ensuring id and
+        # type are present
+        reasoning_extra = {}
+        for chunk in chunks:
+            if chunk.reasoning and chunk.reasoning.extra:
+                reasoning_extra.update(chunk.reasoning.extra)
+        # Ensure id and type are always set, but don't override if already present
+        reasoning_extra.setdefault("id", reasoning_id)
+        reasoning_extra.setdefault("type", "reasoning")
+        reasoning = ReasoningContent(reasoning_text=reasoning_text, extra=reasoning_extra)
 
     return ChatMessage.from_assistant(
         text=text or None, tool_calls=tool_calls, meta=final_response, reasoning=reasoning

releasenotes/notes/fix-openai-responses-streaming-encrypted-content-dropped.yaml

@@ -0,0 +1,9 @@
+---
+fixes:
+  - |
+    Fixed ``OpenAIResponsesChatGenerator`` dropping ``encrypted_content`` from reasoning items in streaming mode.
+    Two issues were addressed: ``_convert_streaming_chunks_to_chat_message`` was discarding all reasoning extras
+    except ``id`` and ``type``; and ``_convert_response_chunk_to_streaming_chunk`` had no handler for the
+    ``response.output_item.done`` event for reasoning items, which is the only event that carries
+    ``encrypted_content``. Both are now fixed, so ``encrypted_content`` is preserved end-to-end and
+    multi-turn conversations with ``store=False`` and ``include=["reasoning.encrypted_content"]`` work correctly.

test/components/generators/chat/test_openai_responses_conversion.py

@@ -97,18 +97,16 @@ def openai_responses_streaming_chunks_with_tool_call():
         ),
         StreamingChunk(
             content="",
-            meta={
-                "item": {
+            meta={"received_at": ANY},
+            index=0,
+            reasoning=ReasoningContent(
+                reasoning_text="",
+                extra={
                     "id": "rs_095b57053855eac100690491f54e308196878239be3ba6133c",
                     "summary": [],
                     "type": "reasoning",
                 },
-                "output_index": 0,
-                "sequence_number": 3,
-                "type": "response.output_item.done",
-                "received_at": ANY,
-            },
-            index=0,
+            ),
         ),
         StreamingChunk(
             content="",
@@ -233,7 +231,11 @@ def test_convert_streaming_chunks_to_chat_message_with_tool_call_empty_reasoning
             _content=[
                 ReasoningContent(
                     reasoning_text="",
-                    extra={"id": "rs_095b57053855eac100690491f54e308196878239be3ba6133c", "type": "reasoning"},
+                    extra={
+                        "id": "rs_095b57053855eac100690491f54e308196878239be3ba6133c",
+                        "summary": [],
+                        "type": "reasoning",
+                    },
                 ),
                 ToolCall(
                     tool_name="weather",
@@ -609,18 +611,16 @@ def test_convert_only_text(self):
             ),
             StreamingChunk(
                 content="",
-                meta={
-                    "received_at": ANY,
-                    "item": {
+                meta={"received_at": ANY},
+                index=0,
+                reasoning=ReasoningContent(
+                    reasoning_text="",
+                    extra={
                         "id": "rs_0a8811e62a95217b00690c5ff70a308195a8207d7eb43f1d5b",
                         "summary": [],
                         "type": "reasoning",
                     },
-                    "output_index": 0,
-                    "sequence_number": 3,
-                    "type": "response.output_item.done",
-                },
-                index=0,
+                ),
             ),
             StreamingChunk(
                 content="",
@@ -1075,18 +1075,16 @@ def test_convert_only_function_call(self):
             ),
             StreamingChunk(
                 content="",
-                meta={
-                    "item": {
+                meta={"received_at": ANY},
+                index=0,
+                reasoning=ReasoningContent(
+                    reasoning_text="",
+                    extra={
                         "id": "rs_095b57053855eac100690491f54e308196878239be3ba6133c",
                         "summary": [],
                         "type": "reasoning",
                     },
-                    "output_index": 0,
-                    "sequence_number": 3,
-                    "type": "response.output_item.done",
-                    "received_at": ANY,
-                },
-                index=0,
+                ),
             ),
             StreamingChunk(
                 content="",
@@ -1437,3 +1435,110 @@ def test_convert_invalid(self):
         )
         with pytest.raises(ValueError):
             _convert_chat_message_to_responses_api_format(message)
+
+    def test_convert_streaming_chunks_to_chat_message_preserves_encrypted_content(self):
+        """Test that encrypted_content in reasoning extra is preserved during streaming conversion."""
+        chunks = [
+            StreamingChunk(
+                content="",
+                meta={"received_at": ANY},
+                index=0,
+                start=True,
+                reasoning=ReasoningContent(
+                    reasoning_text="",
+                    extra={
+                        "id": "rs_095b57053855eac100690491f54e308196878239be3ba6133c",
+                        "type": "reasoning",
+                        "encrypted_content": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...",  # Simulated encrypted reasoning
+                        "status": "in_progress",
+                    },
+                ),
+            ),
+            StreamingChunk(
+                content="",
+                meta={
+                    "received_at": ANY,
+                    "response": {
+                        "id": "resp_095b57053855eac100690491f4e22c8196ac124365e8c70424",
+                        "created_at": 1761907188.0,
+                        "model": "gpt-5-mini-2025-08-07",
+                        "object": "response",
+                        "output": [
+                            {
+                                "id": "rs_095b57053855eac100690491f54e308196878239be3ba6133c",
+                                "type": "reasoning",
+                                "encrypted_content": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...",
+                                "status": "completed",
+                            }
+                        ],
+                    },
+                    "sequence_number": 16,
+                    "type": "response.completed",
+                },
+                finish_reason="stop",
+            ),
+        ]
+
+        message = _convert_streaming_chunks_to_chat_message(chunks)
+
+        # Verify reasoning content exists and has the correct structure
+        assert message.reasoning is not None
+        assert message.reasoning.reasoning_text == ""
+
+        # Verify encrypted_content is preserved along with id and type
+        assert message.reasoning.extra.get("id") == "rs_095b57053855eac100690491f54e308196878239be3ba6133c"
+        assert message.reasoning.extra.get("type") == "reasoning"
+        assert message.reasoning.extra.get("encrypted_content") == "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9..."
+        assert message.reasoning.extra.get("status") == "in_progress"
+
+    def test_encrypted_content_preserved_through_full_streaming_pipeline(self):
+        """
+        Feeds real OpenAI event objects through the full pipeline:
+
+         _convert_response_chunk_to_streaming_chunk → _convert_streaming_chunks_to_chat_message
+
+        to verify encrypted_content survives end-to-end.
+        """
+        REASONING_ID = "rs_abc123"
+        ENCRYPTED = "eyJhbGciOiJIUzI1NiJ9.encrypted_reasoning"
+
+        openai_events = [
+            # reasoning item starts — encrypted_content not yet available
+            ResponseOutputItemAddedEvent(
+                item=ResponseReasoningItem(id=REASONING_ID, summary=[], type="reasoning", status="in_progress"),
+                output_index=0,
+                sequence_number=0,
+                type="response.output_item.added",
+            ),
+            # reasoning item finishes — encrypted_content is now populated
+            ResponseOutputItemDoneEvent(
+                item=ResponseReasoningItem(
+                    id=REASONING_ID, summary=[], type="reasoning", encrypted_content=ENCRYPTED, status="completed"
+                ),
+                output_index=0,
+                sequence_number=1,
+                type="response.output_item.done",
+            ),
+        ]
+
+        streaming_chunks = []
+        for event in openai_events:
+            chunk = _convert_response_chunk_to_streaming_chunk(event, previous_chunks=streaming_chunks)
+            streaming_chunks.append(chunk)
+
+        # The done chunk must carry reasoning so encrypted_content reaches the assembly step
+        done_chunk = streaming_chunks[1]
+        assert done_chunk.reasoning is not None, (
+            "response.output_item.done for reasoning must produce a StreamingChunk with reasoning set; "
+            "without this, encrypted_content is silently dropped before assembly"
+        )
+        assert done_chunk.reasoning.extra.get("encrypted_content") == ENCRYPTED
+
+        message = _convert_streaming_chunks_to_chat_message(streaming_chunks)
+
+        assert message.reasoning is not None
+        assert message.reasoning.extra.get("id") == REASONING_ID
+        assert message.reasoning.extra.get("encrypted_content") == ENCRYPTED, (
+            "encrypted_content was dropped — the response.output_item.done event for reasoning items "
+            "must be handled in _convert_response_chunk_to_streaming_chunk"
+        )