Revert "fix: updating generated answer and extracted answers serialisation (#11805)" (#11895)

Author: sjrl

haystack/dataclasses/answer.py

@@ -5,6 +5,7 @@
 from dataclasses import asdict, dataclass, field
 from typing import Any, Optional, Protocol, runtime_checkable
 
+from haystack.core.serialization import default_from_dict, default_to_dict
 from haystack.dataclasses import ChatMessage, Document
 from haystack.utils.dataclasses import _warn_on_inplace_mutation
 
@@ -53,16 +54,20 @@ def to_dict(self) -> dict[str, Any]:
         :returns:
             Serialized dictionary representation of the object.
         """
-        return {
-            "data": self.data,
-            "query": self.query,
-            "document": self.document.to_dict(flatten=False) if self.document is not None else None,
-            "context": self.context,
-            "score": self.score,
-            "document_offset": asdict(self.document_offset) if self.document_offset is not None else None,
-            "context_offset": asdict(self.context_offset) if self.context_offset is not None else None,
-            "meta": self.meta,
-        }
+        document = self.document.to_dict(flatten=False) if self.document is not None else None
+        document_offset = asdict(self.document_offset) if self.document_offset is not None else None
+        context_offset = asdict(self.context_offset) if self.context_offset is not None else None
+        return default_to_dict(
+            self,
+            data=self.data,
+            query=self.query,
+            document=document,
+            context=self.context,
+            score=self.score,
+            document_offset=document_offset,
+            context_offset=context_offset,
+            meta=self.meta,
+        )
 
     @classmethod
     def from_dict(cls, data: dict[str, Any]) -> "ExtractedAnswer":
@@ -74,32 +79,16 @@ def from_dict(cls, data: dict[str, Any]) -> "ExtractedAnswer":
         :returns:
             Deserialized object.
         """
-        # backward compat: old format wrapped fields in init_parameters
-        if "init_parameters" in data:
-            data = data["init_parameters"]
-
-        document = data.get("document")
-        if document is not None:
-            document = Document.from_dict(document)
-
-        document_offset = data.get("document_offset")
-        if document_offset is not None:
-            document_offset = ExtractedAnswer.Span(**document_offset)
-
-        context_offset = data.get("context_offset")
-        if context_offset is not None:
-            context_offset = ExtractedAnswer.Span(**context_offset)
-
-        return cls(
-            data=data.get("data"),
-            query=data["query"],
-            score=data["score"],
-            document=document,
-            context=data.get("context"),
-            document_offset=document_offset,
-            context_offset=context_offset,
-            meta=data.get("meta", {}),
-        )
+        init_params = data.get("init_parameters", {})
+        if (doc := init_params.get("document")) is not None:
+            data["init_parameters"]["document"] = Document.from_dict(doc)
+
+        if (offset := init_params.get("document_offset")) is not None:
+            data["init_parameters"]["document_offset"] = ExtractedAnswer.Span(**offset)
+
+        if (offset := init_params.get("context_offset")) is not None:
+            data["init_parameters"]["context_offset"] = ExtractedAnswer.Span(**offset)
+        return default_from_dict(cls, data)
 
 
 @_warn_on_inplace_mutation
@@ -121,18 +110,17 @@ def to_dict(self) -> dict[str, Any]:
         :returns:
             Serialized dictionary representation of the object.
         """
-        # all_messages is either a list of ChatMessage objects or a list of strings
+        documents = [doc.to_dict(flatten=False) for doc in self.documents]
+
+        # Serialize ChatMessage objects to dicts
         meta = self.meta
         all_messages = meta.get("all_messages")
+
+        # all_messages is either a list of ChatMessage objects or a list of strings
         if all_messages and isinstance(all_messages[0], ChatMessage):
             meta = {**meta, "all_messages": [msg.to_dict() for msg in all_messages]}
 
-        return {
-            "data": self.data,
-            "query": self.query,
-            "documents": [doc.to_dict(flatten=False) for doc in self.documents],
-            "meta": meta,
-        }
+        return default_to_dict(self, data=self.data, query=self.query, documents=documents, meta=meta)
 
     @classmethod
     def from_dict(cls, data: dict[str, Any]) -> "GeneratedAnswer":
@@ -145,15 +133,14 @@ def from_dict(cls, data: dict[str, Any]) -> "GeneratedAnswer":
         :returns:
             Deserialized object.
         """
-        # backward compatibility: old format wrapped fields in init_parameters
-        if "init_parameters" in data:
-            data = data["init_parameters"]
+        init_params = data.get("init_parameters", {})
 
-        documents = [Document.from_dict(d) for d in data.get("documents", [])]
+        if (documents := init_params.get("documents")) is not None:
+            init_params["documents"] = [Document.from_dict(d) for d in documents]
 
-        # copy to avoid mutating the caller's input dict when converting all_messages
-        meta = dict(data.get("meta", {}))
+        meta = init_params.get("meta", {})
         if (all_messages := meta.get("all_messages")) and isinstance(all_messages[0], dict):
             meta["all_messages"] = [ChatMessage.from_dict(m) for m in all_messages]
+        init_params["meta"] = meta
 
-        return cls(data=data["data"], query=data["query"], documents=documents, meta=meta)
+        return default_from_dict(cls, data)

releasenotes/notes/fix-dataclasses-deseria-0bb839b5edd3fca6.yaml

@@ -57,17 +57,20 @@ def test_to_dict(self):
             meta={"meta_key": "meta_value"},
         )
         assert answer.to_dict() == {
-            "data": "42",
-            "query": "What is the answer?",
-            "document": document.to_dict(flatten=False),
-            "context": "The answer is 42.",
-            "score": 1.0,
-            "document_offset": {"start": 42, "end": 44},
-            "context_offset": {"start": 14, "end": 16},
-            "meta": {"meta_key": "meta_value"},
+            "type": "haystack.dataclasses.answer.ExtractedAnswer",
+            "init_parameters": {
+                "data": "42",
+                "query": "What is the answer?",
+                "document": document.to_dict(flatten=False),
+                "context": "The answer is 42.",
+                "score": 1.0,
+                "document_offset": {"start": 42, "end": 44},
+                "context_offset": {"start": 14, "end": 16},
+                "meta": {"meta_key": "meta_value"},
+            },
         }
 
-    def test_from_dict_legacy(self):
+    def test_from_dict(self):
         answer = ExtractedAnswer.from_dict(
             {
                 "type": "haystack.dataclasses.answer.ExtractedAnswer",
@@ -98,34 +101,6 @@ def test_from_dict_legacy(self):
         assert answer.context_offset == ExtractedAnswer.Span(14, 16)
         assert answer.meta == {"meta_key": "meta_value"}
 
-    def test_from_dict(self):
-        answer = ExtractedAnswer.from_dict(
-            {
-                "data": "42",
-                "query": "What is the answer?",
-                "document": {
-                    "id": "8f800a524b139484fc719ecc35f971a080de87618319bc4836b784d69baca57f",
-                    "content": "I thought a lot about this. The answer is 42.",
-                },
-                "context": "The answer is 42.",
-                "score": 1.0,
-                "document_offset": {"start": 42, "end": 44},
-                "context_offset": {"start": 14, "end": 16},
-                "meta": {"meta_key": "meta_value"},
-            }
-        )
-        assert answer.data == "42"
-        assert answer.query == "What is the answer?"
-        assert answer.document == Document(
-            id="8f800a524b139484fc719ecc35f971a080de87618319bc4836b784d69baca57f",
-            content="I thought a lot about this. The answer is 42.",
-        )
-        assert answer.context == "The answer is 42."
-        assert answer.score == 1.0
-        assert answer.document_offset == ExtractedAnswer.Span(42, 44)
-        assert answer.context_offset == ExtractedAnswer.Span(14, 16)
-        assert answer.meta == {"meta_key": "meta_value"}
-
     def test_no_warning_on_init(self):
         with warnings.catch_warnings():
             warnings.simplefilter("error", Warning)
@@ -183,7 +158,10 @@ def test_protocol(self):
 
     def test_to_dict(self):
         answer = GeneratedAnswer(data="42", query="What is the answer?", documents=[])
-        assert answer.to_dict() == {"data": "42", "query": "What is the answer?", "documents": [], "meta": {}}
+        assert answer.to_dict() == {
+            "type": "haystack.dataclasses.answer.GeneratedAnswer",
+            "init_parameters": {"data": "42", "query": "What is the answer?", "documents": [], "meta": {}},
+        }
 
     def test_to_dict_with_meta(self):
         answer = GeneratedAnswer(
@@ -193,10 +171,13 @@ def test_to_dict_with_meta(self):
             meta={"meta_key": "meta_value", "all_messages": ["What is the answer?"]},
         )
         assert answer.to_dict() == {
-            "data": "42",
-            "query": "What is the answer?",
-            "documents": [],
-            "meta": {"meta_key": "meta_value", "all_messages": ["What is the answer?"]},
+            "type": "haystack.dataclasses.answer.GeneratedAnswer",
+            "init_parameters": {
+                "data": "42",
+                "query": "What is the answer?",
+                "documents": [],
+                "meta": {"meta_key": "meta_value", "all_messages": ["What is the answer?"]},
+            },
         }
 
     def test_to_dict_with_chat_message_in_meta(self):
@@ -212,16 +193,19 @@ def test_to_dict_with_chat_message_in_meta(self):
             meta={"meta_key": "meta_value", "all_messages": [ChatMessage.from_user("What is the answer?")]},
         )
         assert answer.to_dict() == {
-            "data": "42",
-            "query": "What is the answer?",
-            "documents": [d.to_dict(flatten=False) for d in documents],
-            "meta": {
-                "meta_key": "meta_value",
-                "all_messages": [ChatMessage.from_user("What is the answer?").to_dict()],
+            "type": "haystack.dataclasses.answer.GeneratedAnswer",
+            "init_parameters": {
+                "data": "42",
+                "query": "What is the answer?",
+                "documents": [d.to_dict(flatten=False) for d in documents],
+                "meta": {
+                    "meta_key": "meta_value",
+                    "all_messages": [ChatMessage.from_user("What is the answer?").to_dict()],
+                },
             },
         }
 
-    def test_from_dict_legacy(self):
+    def test_from_dict(self):
         answer = GeneratedAnswer.from_dict(
             {
                 "type": "haystack.dataclasses.answer.GeneratedAnswer",
@@ -233,7 +217,7 @@ def test_from_dict_legacy(self):
         assert answer.documents == []
         assert answer.meta == {}
 
-    def test_from_dict_with_meta_legacy(self):
+    def test_from_dict_with_meta(self):
         answer = GeneratedAnswer.from_dict(
             {
                 "type": "haystack.dataclasses.answer.GeneratedAnswer",
@@ -251,7 +235,7 @@ def test_from_dict_with_meta_legacy(self):
         assert answer.meta["meta_key"] == "meta_value"
         assert answer.meta["all_messages"] == ["What is the answer?"]
 
-    def test_from_dict_with_chat_message_in_meta_legacy(self):
+    def test_from_dict_with_chat_message_in_meta(self):
         answer = GeneratedAnswer.from_dict(
             {
                 "type": "haystack.dataclasses.answer.GeneratedAnswer",
@@ -280,7 +264,7 @@ def test_from_dict_with_chat_message_in_meta_legacy(self):
         assert answer.meta["meta_key"] == "meta_value"
         assert answer.meta["all_messages"] == [ChatMessage.from_user("What is the answer?")]
 
-    def test_from_dict_with_empty_all_messages_legacy(self):
+    def test_from_dict_with_empty_all_messages(self):
         # An empty `all_messages` list must not crash deserialization: `is not None`
         # let `[]` through and then indexed `all_messages[0]`, raising IndexError.
         answer = GeneratedAnswer.from_dict(
@@ -296,73 +280,6 @@ def test_from_dict_with_empty_all_messages_legacy(self):
         )
         assert answer.meta["all_messages"] == []
 
-    def test_from_dict(self):
-        answer = GeneratedAnswer.from_dict({"data": "42", "query": "What is the answer?", "documents": [], "meta": {}})
-        assert answer.data == "42"
-        assert answer.query == "What is the answer?"
-        assert answer.documents == []
-        assert answer.meta == {}
-
-    def test_from_dict_with_meta(self):
-        answer = GeneratedAnswer.from_dict(
-            {
-                "data": "42",
-                "query": "What is the answer?",
-                "documents": [],
-                "meta": {"meta_key": "meta_value", "all_messages": ["What is the answer?"]},
-            }
-        )
-        assert answer.data == "42"
-        assert answer.query == "What is the answer?"
-        assert answer.documents == []
-        assert answer.meta["meta_key"] == "meta_value"
-        assert answer.meta["all_messages"] == ["What is the answer?"]
-
-    def test_from_dict_with_chat_message_in_meta(self):
-        answer = GeneratedAnswer.from_dict(
-            {
-                "data": "42",
-                "query": "What is the answer?",
-                "documents": [
-                    {"id": "1", "content": "The answer is 42."},
-                    {"id": "2", "content": "I believe the answer is 42."},
-                    {"id": "3", "content": "42 is definitely the answer."},
-                ],
-                "meta": {
-                    "meta_key": "meta_value",
-                    "all_messages": [ChatMessage.from_user("What is the answer?").to_dict()],
-                },
-            }
-        )
-        assert answer.data == "42"
-        assert answer.query == "What is the answer?"
-        assert answer.documents == [
-            Document(id="1", content="The answer is 42."),
-            Document(id="2", content="I believe the answer is 42."),
-            Document(id="3", content="42 is definitely the answer."),
-        ]
-        assert answer.meta["meta_key"] == "meta_value"
-        assert answer.meta["all_messages"] == [ChatMessage.from_user("What is the answer?")]
-
-    def test_from_dict_with_empty_all_messages(self):
-        answer = GeneratedAnswer.from_dict(
-            {"data": "42", "query": "What is the answer?", "documents": [], "meta": {"all_messages": []}}
-        )
-        assert answer.meta["all_messages"] == []
-
-    def test_from_dict_does_not_mutate_input(self):
-        # from_dict must not mutate the caller's input dict while converting
-        # `all_messages` dicts into ChatMessage objects (regression test).
-        meta = {"all_messages": [ChatMessage.from_user("What is the answer?").to_dict()]}
-        serialized = {"data": "42", "query": "What is the answer?", "documents": [], "meta": meta}
-        answer = GeneratedAnswer.from_dict(serialized)
-
-        # the deserialized answer still holds ChatMessage objects
-        assert answer.meta["all_messages"] == [ChatMessage.from_user("What is the answer?")]
-        # but the caller's meta dict is left untouched: it still holds plain dicts
-        assert isinstance(meta["all_messages"][0], dict)
-        assert meta["all_messages"] == [ChatMessage.from_user("What is the answer?").to_dict()]
-
     def test_to_dict_from_dict_round_trip_with_empty_all_messages(self):
         answer = GeneratedAnswer(data="42", query="What is the answer?", documents=[], meta={"all_messages": []})
         assert GeneratedAnswer.from_dict(answer.to_dict()) == answer