diff --git a/MIGRATION.md b/MIGRATION.md index 7b72c178be..c98025f260 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -32,6 +32,34 @@ GeneratedAnswer.from_dict(serialized) # new flat format GeneratedAnswer.from_dict(old_wrapped_dict) # old {"type": ..., "init_parameters": {...}} format ``` +### Pydantic serialization of `ChatMessage` + +**What changed:** `ChatMessage` now defines its own Pydantic schema. Pydantic models with `ChatMessage` fields serialize with `ChatMessage.to_dict` and validate dictionaries with `ChatMessage.from_dict`. As a result, `model_dump()` produces the canonical serialization format (`role`/`content` keys) instead of the raw dataclass fields (`_role`/`_content` keys). + +**Why:** Pydantic used to auto-serialize `ChatMessage` as a plain dataclass, producing an internal format that diverged from `to_dict` and could not be fully deserialized. Emitting the canonical format lets messages embedded in Pydantic models (such as API request/response DTOs) round-trip cleanly across services. + +**Deserialization is backward compatible:** `ChatMessage.from_dict` and Pydantic validation accept both the canonical format and the old raw dataclass format, so payloads produced by older versions keep deserializing without changes. + +**How to migrate:** Only code that reads Pydantic-dumped messages by key needs updating: access the canonical keys instead of the raw dataclass fields. + +Before (v2.x): +```python +from pydantic import BaseModel +from haystack.dataclasses import ChatMessage + +class Response(BaseModel): + messages: list[ChatMessage] + +dumped = Response(messages=[ChatMessage.from_user("Hi")]).model_dump() +role = dumped["messages"][0]["_role"] # raw dataclass fields +``` + +After (v3.0): +```python +dumped = Response(messages=[ChatMessage.from_user("Hi")]).model_dump() +role = dumped["messages"][0]["role"] # canonical ChatMessage.to_dict format +``` + ### Components Moved to External Packages **What changed:** Some components have been moved out of Haystack into dedicated integration packages, diff --git a/haystack/dataclasses/chat_message.py b/haystack/dataclasses/chat_message.py index 4e619a90eb..a843468f36 100644 --- a/haystack/dataclasses/chat_message.py +++ b/haystack/dataclasses/chat_message.py @@ -8,6 +8,9 @@ from enum import Enum from typing import Any +from pydantic import GetCoreSchemaHandler +from pydantic_core import CoreSchema, core_schema + from haystack import logging from haystack.dataclasses.file_content import FileContent from haystack.dataclasses.image_content import ImageContent @@ -279,6 +282,21 @@ def _serialize_content_part(part: ChatMessageContentT) -> dict[str, Any]: return {serialization_key: part.to_dict()} +def _validate_chat_message(value: Any) -> "ChatMessage": + """ + Coerce a value into a ChatMessage during Pydantic validation. + + :param value: A ChatMessage instance or a dictionary in a format accepted by `ChatMessage.from_dict`. + :returns: A ChatMessage object. + :raises ValueError: If the value is neither a ChatMessage nor a dictionary. + """ + if isinstance(value, ChatMessage): + return value + if isinstance(value, dict): + return ChatMessage.from_dict(value) + raise ValueError(f"Expected `ChatMessage` or `dict`, got `{type(value).__name__}`") + + @_warn_on_inplace_mutation @dataclass class ChatMessage: @@ -630,6 +648,19 @@ def from_dict(cls, data: dict[str, Any]) -> "ChatMessage": raise ValueError(f"Missing 'content' or '_content' in serialized ChatMessage: `{data}`") + @classmethod + def __get_pydantic_core_schema__(cls, source_type: Any, handler: GetCoreSchemaHandler) -> CoreSchema: + """ + Defines how Pydantic validates and serializes ChatMessage. + + Dictionaries are validated with `from_dict` and existing instances pass through unchanged. + Serialization uses `to_dict`, so Pydantic's `model_dump` produces the canonical format instead + of the raw dataclass fields. + """ + return core_schema.no_info_plain_validator_function( + _validate_chat_message, serialization=core_schema.plain_serializer_function_ser_schema(cls.to_dict) + ) + def to_openai_dict_format(self, require_tool_call_ids: bool = True) -> dict[str, Any]: """ Convert a ChatMessage to the dictionary format expected by OpenAI's Chat Completions API. diff --git a/releasenotes/notes/chatmessage-pydantic-integration-41204145db28d04d.yaml b/releasenotes/notes/chatmessage-pydantic-integration-41204145db28d04d.yaml new file mode 100644 index 0000000000..699dce0bf0 --- /dev/null +++ b/releasenotes/notes/chatmessage-pydantic-integration-41204145db28d04d.yaml @@ -0,0 +1,14 @@ +--- +upgrade: + - | + Pydantic's ``model_dump`` on models containing ``ChatMessage`` fields now returns the canonical + ``ChatMessage.to_dict`` format (``role``/``content`` keys) instead of the raw dataclass fields + (``_role``/``_content`` keys). ``ChatMessage.from_dict`` accepts both formats, so round-tripping + existing payloads keeps working. If you post-process dumped messages and rely on the raw dataclass + keys, update your code to the canonical format. +enhancements: + - | + ``ChatMessage`` now integrates natively with Pydantic. Pydantic models with ``ChatMessage`` fields + serialize with ``ChatMessage.to_dict`` and validate dictionaries with ``ChatMessage.from_dict``, so + ``model_dump`` produces the canonical serialization format and serialized messages round-trip + through Pydantic models without custom serializers or validators. diff --git a/test/dataclasses/test_chat_message.py b/test/dataclasses/test_chat_message.py index 279e8c5100..4b11e9dae9 100644 --- a/test/dataclasses/test_chat_message.py +++ b/test/dataclasses/test_chat_message.py @@ -8,7 +8,7 @@ from typing import Any import pytest -from pydantic import BaseModel +from pydantic import BaseModel, ValidationError from haystack.dataclasses.chat_message import ( ChatMessage, @@ -764,13 +764,33 @@ class MessageEnvelope(BaseModel): class TestFromDictPydanticDump: """ - `ChatMessage.from_dict` supports the format Pydantic produces when it auto-serializes ChatMessage as a plain - dataclass: raw dataclass fields (`_role`, `_content`, ...) with unwrapped content parts. + `ChatMessage.from_dict` supports the formats Pydantic produces for ChatMessage fields: the canonical + `to_dict` format (what `model_dump` produces now that ChatMessage defines its own Pydantic schema) and + the raw dataclass format with unwrapped content parts that older versions produced. """ def _pydantic_dump(self, message: ChatMessage) -> dict[str, Any]: return MessageEnvelope(message=message).model_dump(mode="json")["message"] + def test_raw_dataclass_format(self): + # the raw dataclass format that Pydantic produced before ChatMessage defined its own Pydantic schema + data = { + "_role": "assistant", + "_content": [ + {"reasoning_text": "Thinking...", "extra": {}}, + {"text": "Let me check."}, + {"tool_name": "mytool", "arguments": {"a": 1}, "id": "123", "extra": None}, + ], + "_name": None, + "_meta": {"some": "info"}, + } + assert ChatMessage.from_dict(data) == ChatMessage.from_assistant( + "Let me check.", + meta={"some": "info"}, + tool_calls=[ToolCall(tool_name="mytool", arguments={"a": 1}, id="123")], + reasoning=ReasoningContent(reasoning_text="Thinking..."), + ) + def test_text_message(self): message = ChatMessage.from_user("What is the answer?", meta={"some": "info"}, name="virginia") assert ChatMessage.from_dict(self._pydantic_dump(message)) == message @@ -837,6 +857,57 @@ class Response(BaseModel): assert [ChatMessage.from_dict(message) for message in dumped["messages"]] == messages +class ChatHistory(BaseModel): + messages: list[ChatMessage] + + +class TestPydanticIntegration: + def test_model_dump_uses_to_dict(self): + messages = [ + ChatMessage.from_user("Hi"), + ChatMessage.from_assistant(tool_calls=[ToolCall(tool_name="mytool", arguments={"a": 1}, id="123")]), + ] + history = ChatHistory(messages=messages) + assert history.model_dump() == {"messages": [message.to_dict() for message in messages]} + assert history.model_dump(mode="json") == {"messages": [message.to_dict() for message in messages]} + + def test_model_validate_serialized_messages(self): + messages = [ChatMessage.from_user("Hi"), ChatMessage.from_assistant("Hello")] + history = ChatHistory.model_validate({"messages": [message.to_dict() for message in messages]}) + assert history.messages == messages + + def test_instances_pass_through_unchanged(self): + message = ChatMessage.from_user("Hi") + history = ChatHistory(messages=[message]) + assert history.messages[0] is message + + def test_round_trip(self, base64_image_string): + tool_call = ToolCall(id="123", tool_name="mytool", arguments={"a": 1}) + messages = [ + ChatMessage.from_system("You are helpful."), + ChatMessage.from_assistant( + "Checking.", + meta={"some": "info"}, + tool_calls=[tool_call], + reasoning=ReasoningContent(reasoning_text="Thinking..."), + ), + ChatMessage.from_tool(tool_result="42", origin=tool_call), + ChatMessage.from_user( + content_parts=[ + ImageContent(base64_image=base64_image_string, mime_type="image/png"), + FileContent(base64_data="aGVsbG8=", mime_type="text/plain", filename="hello.txt"), + ] + ), + ] + + dumped = ChatHistory(messages=messages).model_dump(mode="json") + assert ChatHistory.model_validate(dumped).messages == messages + + def test_invalid_value_raises_validation_error(self): + with pytest.raises(ValidationError): + ChatHistory(messages=["not a message"]) # type: ignore[list-item] + + class TestToOpenaiDictFormat: def test_to_openai_dict_format_system_message(self): message = ChatMessage.from_system("You are good assistant")