feat!: Update ChatMessage to use to_dict and from_dict in pydantic serialization

MIGRATION.md

@@ -32,6 +32,34 @@ GeneratedAnswer.from_dict(serialized)          # new flat format
 GeneratedAnswer.from_dict(old_wrapped_dict)    # old {"type": ..., "init_parameters": {...}} format
 ```
 
+### Pydantic serialization of `ChatMessage`
+
+**What changed:** `ChatMessage` now defines its own Pydantic schema. Pydantic models with `ChatMessage` fields serialize with `ChatMessage.to_dict` and validate dictionaries with `ChatMessage.from_dict`. As a result, `model_dump()` produces the canonical serialization format (`role`/`content` keys) instead of the raw dataclass fields (`_role`/`_content` keys).
+
+**Why:** Pydantic used to auto-serialize `ChatMessage` as a plain dataclass, producing an internal format that diverged from `to_dict` and could not be fully deserialized. Emitting the canonical format lets messages embedded in Pydantic models (such as API request/response DTOs) round-trip cleanly across services.
+
+**Deserialization is backward compatible:** `ChatMessage.from_dict` and Pydantic validation accept both the canonical format and the old raw dataclass format, so payloads produced by older versions keep deserializing without changes.
+
+**How to migrate:** Only code that reads Pydantic-dumped messages by key needs updating: access the canonical keys instead of the raw dataclass fields.
+
+Before (v2.x):
+```python
+from pydantic import BaseModel
+from haystack.dataclasses import ChatMessage
+
+class Response(BaseModel):
+    messages: list[ChatMessage]
+
+dumped = Response(messages=[ChatMessage.from_user("Hi")]).model_dump()
+role = dumped["messages"][0]["_role"]  # raw dataclass fields
+```
+
+After (v3.0):
+```python
+dumped = Response(messages=[ChatMessage.from_user("Hi")]).model_dump()
+role = dumped["messages"][0]["role"]  # canonical ChatMessage.to_dict format
+```
+
 ### Components Moved to External Packages
 
 **What changed:** Some components have been moved out of Haystack into dedicated integration packages,

haystack/dataclasses/chat_message.py

@@ -8,6 +8,9 @@
 from enum import Enum
 from typing import Any
 
+from pydantic import GetCoreSchemaHandler
+from pydantic_core import CoreSchema, core_schema
+
 from haystack import logging
 from haystack.dataclasses.file_content import FileContent
 from haystack.dataclasses.image_content import ImageContent
@@ -279,6 +282,21 @@ def _serialize_content_part(part: ChatMessageContentT) -> dict[str, Any]:
     return {serialization_key: part.to_dict()}
 
 
+def _validate_chat_message(value: Any) -> "ChatMessage":
+    """
+    Coerce a value into a ChatMessage during Pydantic validation.
+
+    :param value: A ChatMessage instance or a dictionary in a format accepted by `ChatMessage.from_dict`.
+    :returns: A ChatMessage object.
+    :raises ValueError: If the value is neither a ChatMessage nor a dictionary.
+    """
+    if isinstance(value, ChatMessage):
+        return value
+    if isinstance(value, dict):
+        return ChatMessage.from_dict(value)
+    raise ValueError(f"Expected `ChatMessage` or `dict`, got `{type(value).__name__}`")
+
+
 @_warn_on_inplace_mutation
 @dataclass
 class ChatMessage:
@@ -630,6 +648,19 @@ def from_dict(cls, data: dict[str, Any]) -> "ChatMessage":
 
         raise ValueError(f"Missing 'content' or '_content' in serialized ChatMessage: `{data}`")
 
+    @classmethod
+    def __get_pydantic_core_schema__(cls, source_type: Any, handler: GetCoreSchemaHandler) -> CoreSchema:
+        """
+        Defines how Pydantic validates and serializes ChatMessage.
+
+        Dictionaries are validated with `from_dict` and existing instances pass through unchanged.
+        Serialization uses `to_dict`, so Pydantic's `model_dump` produces the canonical format instead
+        of the raw dataclass fields.
+        """
+        return core_schema.no_info_plain_validator_function(
+            _validate_chat_message, serialization=core_schema.plain_serializer_function_ser_schema(cls.to_dict)
+        )
+
     def to_openai_dict_format(self, require_tool_call_ids: bool = True) -> dict[str, Any]:
         """
         Convert a ChatMessage to the dictionary format expected by OpenAI's Chat Completions API.

releasenotes/notes/chatmessage-pydantic-integration-41204145db28d04d.yaml

@@ -0,0 +1,14 @@
+---
+upgrade:
+  - |
+    Pydantic's ``model_dump`` on models containing ``ChatMessage`` fields now returns the canonical
+    ``ChatMessage.to_dict`` format (``role``/``content`` keys) instead of the raw dataclass fields
+    (``_role``/``_content`` keys). ``ChatMessage.from_dict`` accepts both formats, so round-tripping
+    existing payloads keeps working. If you post-process dumped messages and rely on the raw dataclass
+    keys, update your code to the canonical format.
+enhancements:
+  - |
+    ``ChatMessage`` now integrates natively with Pydantic. Pydantic models with ``ChatMessage`` fields
+    serialize with ``ChatMessage.to_dict`` and validate dictionaries with ``ChatMessage.from_dict``, so
+    ``model_dump`` produces the canonical serialization format and serialized messages round-trip
+    through Pydantic models without custom serializers or validators.

test/dataclasses/test_chat_message.py

@@ -8,7 +8,7 @@
 from typing import Any
 
 import pytest
-from pydantic import BaseModel
+from pydantic import BaseModel, ValidationError
 
 from haystack.dataclasses.chat_message import (
     ChatMessage,
@@ -764,13 +764,33 @@ class MessageEnvelope(BaseModel):
 
 class TestFromDictPydanticDump:
     """
-    `ChatMessage.from_dict` supports the format Pydantic produces when it auto-serializes ChatMessage as a plain
-    dataclass: raw dataclass fields (`_role`, `_content`, ...) with unwrapped content parts.
+    `ChatMessage.from_dict` supports the formats Pydantic produces for ChatMessage fields: the canonical
+    `to_dict` format (what `model_dump` produces now that ChatMessage defines its own Pydantic schema) and
+    the raw dataclass format with unwrapped content parts that older versions produced.
     """
 
     def _pydantic_dump(self, message: ChatMessage) -> dict[str, Any]:
         return MessageEnvelope(message=message).model_dump(mode="json")["message"]
 
+    def test_raw_dataclass_format(self):
+        # the raw dataclass format that Pydantic produced before ChatMessage defined its own Pydantic schema
+        data = {
+            "_role": "assistant",
+            "_content": [
+                {"reasoning_text": "Thinking...", "extra": {}},
+                {"text": "Let me check."},
+                {"tool_name": "mytool", "arguments": {"a": 1}, "id": "123", "extra": None},
+            ],
+            "_name": None,
+            "_meta": {"some": "info"},
+        }
+        assert ChatMessage.from_dict(data) == ChatMessage.from_assistant(
+            "Let me check.",
+            meta={"some": "info"},
+            tool_calls=[ToolCall(tool_name="mytool", arguments={"a": 1}, id="123")],
+            reasoning=ReasoningContent(reasoning_text="Thinking..."),
+        )
+
     def test_text_message(self):
         message = ChatMessage.from_user("What is the answer?", meta={"some": "info"}, name="virginia")
         assert ChatMessage.from_dict(self._pydantic_dump(message)) == message
@@ -837,6 +857,57 @@ class Response(BaseModel):
         assert [ChatMessage.from_dict(message) for message in dumped["messages"]] == messages
 
 
+class ChatHistory(BaseModel):
+    messages: list[ChatMessage]
+
+
+class TestPydanticIntegration:
+    def test_model_dump_uses_to_dict(self):
+        messages = [
+            ChatMessage.from_user("Hi"),
+            ChatMessage.from_assistant(tool_calls=[ToolCall(tool_name="mytool", arguments={"a": 1}, id="123")]),
+        ]
+        history = ChatHistory(messages=messages)
+        assert history.model_dump() == {"messages": [message.to_dict() for message in messages]}
+        assert history.model_dump(mode="json") == {"messages": [message.to_dict() for message in messages]}
+
+    def test_model_validate_serialized_messages(self):
+        messages = [ChatMessage.from_user("Hi"), ChatMessage.from_assistant("Hello")]
+        history = ChatHistory.model_validate({"messages": [message.to_dict() for message in messages]})
+        assert history.messages == messages
+
+    def test_instances_pass_through_unchanged(self):
+        message = ChatMessage.from_user("Hi")
+        history = ChatHistory(messages=[message])
+        assert history.messages[0] is message
+
+    def test_round_trip(self, base64_image_string):
+        tool_call = ToolCall(id="123", tool_name="mytool", arguments={"a": 1})
+        messages = [
+            ChatMessage.from_system("You are helpful."),
+            ChatMessage.from_assistant(
+                "Checking.",
+                meta={"some": "info"},
+                tool_calls=[tool_call],
+                reasoning=ReasoningContent(reasoning_text="Thinking..."),
+            ),
+            ChatMessage.from_tool(tool_result="42", origin=tool_call),
+            ChatMessage.from_user(
+                content_parts=[
+                    ImageContent(base64_image=base64_image_string, mime_type="image/png"),
+                    FileContent(base64_data="aGVsbG8=", mime_type="text/plain", filename="hello.txt"),
+                ]
+            ),
+        ]
+
+        dumped = ChatHistory(messages=messages).model_dump(mode="json")
+        assert ChatHistory.model_validate(dumped).messages == messages
+
+    def test_invalid_value_raises_validation_error(self):
+        with pytest.raises(ValidationError):
+            ChatHistory(messages=["not a message"])  # type: ignore[list-item]
+
+
 class TestToOpenaiDictFormat:
     def test_to_openai_dict_format_system_message(self):
         message = ChatMessage.from_system("You are good assistant")