diff --git a/haystack/utils/base_serialization.py b/haystack/utils/base_serialization.py index 0db98b6d39..43b30947d3 100644 --- a/haystack/utils/base_serialization.py +++ b/haystack/utils/base_serialization.py @@ -28,7 +28,9 @@ def _serialize_value_with_schema(payload: Any) -> dict[str, Any]: # noqa: PLR09 - Objects with to_dict() methods (e.g. dataclasses) - Objects with __dict__ attributes - Dictionaries - - Lists, tuples, and sets. Lists with mixed types are not supported. + - Lists, tuples, and sets, including ones holding mixed types. A mixed-type array records one + schema per position under the JSON Schema `prefixItems` keyword, while a homogeneous array + keeps the single shared `items` schema. - Primitive types (str, int, float, bool, None) This is for runtime values (Agent/State data, pipeline inputs/outputs at a breakpoint), not @@ -61,20 +63,23 @@ def _serialize_value_with_schema(payload: Any) -> dict[str, Any]: # noqa: PLR09 # Handle array case - iterate through elements if isinstance(payload, (list, tuple, set, frozenset)): - # Serialize each item in the array + # Serialize each item in the array, keeping its own schema serialized_list = [] + item_schemas: list[Any] = [] + base_schema: dict[str, Any] for item in payload: serialized_value = _serialize_value_with_schema(item) serialized_list.append(serialized_value["serialized_data"]) + item_schemas.append(serialized_value["serialization_schema"]) - # Determine item type from first element (if any) - # NOTE: We do not support mixed-type lists - if payload: - first = next(iter(payload)) - item_schema = _serialize_value_with_schema(first) - base_schema = {"type": "array", "items": item_schema["serialization_schema"]} - else: + if not item_schemas: base_schema = {"type": "array", "items": {}} + elif all(item_schema == item_schemas[0] for item_schema in item_schemas): + # Homogeneous array: keep the historical `items` envelope so older readers keep working + base_schema = {"type": "array", "items": item_schemas[0]} + else: + # Mixed-type array: describe every position with the JSON Schema `prefixItems` keyword + base_schema = {"type": "array", "prefixItems": item_schemas} # Add JSON Schema properties to infer sets, frozensets and tuples if isinstance(payload, (set, frozenset)): @@ -194,7 +199,8 @@ def _deserialize_value_with_schema(serialized: dict[str, Any]) -> Any: "serialized_data": } - NOTE: For array types we only support homogeneous lists (all elements of the same type). + For array types, `prefixItems` (one schema per position, emitted for mixed-type arrays) takes + precedence over `items` (a single shared schema, emitted for homogeneous arrays). :param serialized: The serialized dict with schema and data. :returns: The deserialized value in its original form. @@ -233,10 +239,23 @@ def _deserialize_value_with_schema(serialized: dict[str, Any]) -> Any: # Handle array case if schema_type == "array": - # Deserialize each item + # Mixed-type arrays carry one schema per position under `prefixItems`, + # homogeneous ones carry a single shared schema under `items`. + prefix_items = schema.get("prefixItems") + if prefix_items is not None: + if len(prefix_items) != len(data): + raise DeserializationError( + f"Invalid array payload: 'prefixItems' declares {len(prefix_items)} element schemas " + f"but 'serialized_data' holds {len(data)} elements." + ) + item_schemas: list[Any] = list(prefix_items) + else: + item_schemas = [schema["items"]] * len(data) + + # Deserialize each item with the schema of its own position deserialized_items = [ - _deserialize_value_with_schema({"serialization_schema": schema["items"], "serialized_data": item}) - for item in data + _deserialize_value_with_schema({"serialization_schema": item_schema, "serialized_data": item}) + for item_schema, item in zip(item_schemas, data, strict=True) ] final_array: list | set | frozenset | tuple # Is a set or frozenset if uniqueItems is True diff --git a/releasenotes/notes/fix-mixed-type-lists-schema-serialization-3f1c5a2b7d904e18.yaml b/releasenotes/notes/fix-mixed-type-lists-schema-serialization-3f1c5a2b7d904e18.yaml new file mode 100644 index 0000000000..33827dffe1 --- /dev/null +++ b/releasenotes/notes/fix-mixed-type-lists-schema-serialization-3f1c5a2b7d904e18.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + Fixed schema-based serialization of lists, tuples and sets holding mixed types. Previously the + schema was derived from the first element only, so deserializing such a value raised an + `AttributeError` or silently returned mis-typed data (for example an Agent `State` field or a + pipeline breakpoint input holding `[Document(...), "text", 3]`). Mixed-type arrays now record one + schema per position using the JSON Schema `prefixItems` keyword and round-trip correctly. + Homogeneous arrays keep the exact same output as before, so existing snapshots still load. diff --git a/test/utils/test_base_serialization.py b/test/utils/test_base_serialization.py index 67f48071c0..683f35c4a7 100644 --- a/test/utils/test_base_serialization.py +++ b/test/utils/test_base_serialization.py @@ -373,6 +373,86 @@ def test_serialize_and_deserialize_value_with_schema_with_various_types(): assert _deserialize_value_with_schema(expected) == data +class TestMixedTypeArrays: + """Mixed-type arrays keep one schema per position under `prefixItems`.""" + + def test_serialize_mixed_primitives(self): + assert _serialize_value_with_schema({"y": [1, "a", {"k": 2}]}) == { + "serialization_schema": { + "type": "object", + "properties": { + "y": { + "type": "array", + "prefixItems": [ + {"type": "integer"}, + {"type": "string"}, + {"type": "object", "properties": {"k": {"type": "integer"}}}, + ], + } + }, + }, + "serialized_data": {"y": [1, "a", {"k": 2}]}, + } + + @pytest.mark.parametrize( + "value", + [ + pytest.param({"y": [1, "a", {"k": 2}]}, id="mixed-primitives"), + pytest.param({"x": [Document(content="a", id="1"), "plain string", 3]}, id="object-and-primitives"), + pytest.param({"t": (1, "a", Document(content="a", id="1"))}, id="mixed-tuple"), + pytest.param({"n": [[1, "a"], [Document(content="a", id="1"), None]]}, id="nested-mixed-lists"), + pytest.param({"e": []}, id="empty-list"), + pytest.param({"m": [1, None, True, 2.5, "s"]}, id="all-primitive-kinds"), + ], + ) + def test_round_trip_mixed_arrays(self, value): + deserialized = _deserialize_value_with_schema(_serialize_value_with_schema(value)) + assert deserialized == value + for key, original in value.items(): + assert type(deserialized[key]) is type(original) + + def test_round_trip_mixed_set(self): + value = {1, "a", None} + deserialized = _deserialize_value_with_schema(_serialize_value_with_schema(value)) + assert deserialized == value + assert type(deserialized) is set + + def test_round_trip_mixed_frozenset(self): + value = frozenset({1, "a", None}) + deserialized = _deserialize_value_with_schema(_serialize_value_with_schema(value)) + assert deserialized == value + assert type(deserialized) is frozenset + + def test_homogeneous_output_is_unchanged(self): + # Backward compatibility: homogeneous arrays keep the historical `items` envelope. + document = Document(content="a", id="1") + assert _serialize_value_with_schema({"docs": [document], "nums": (1, 2)}) == { + "serialization_schema": { + "type": "object", + "properties": { + "docs": {"type": "array", "items": {"type": "haystack.dataclasses.document.Document"}}, + "nums": {"type": "array", "items": {"type": "integer"}, "minItems": 2, "maxItems": 2}, + }, + }, + "serialized_data": {"docs": [document.to_dict()], "nums": [1, 2]}, + } + + def test_deserialize_old_format_with_only_items(self): + # Payloads written before `prefixItems` existed still deserialize through `items`. + assert _deserialize_value_with_schema( + {"serialization_schema": {"type": "array", "items": {"type": "integer"}}, "serialized_data": [1, 2, 3]} + ) == [1, 2, 3] + + def test_deserialize_prefix_items_length_mismatch_raises(self): + with pytest.raises(DeserializationError, match="'prefixItems' declares 2 element schemas"): + _deserialize_value_with_schema( + { + "serialization_schema": {"type": "array", "prefixItems": [{"type": "integer"}, {"type": "string"}]}, + "serialized_data": [1], + } + ) + + class TestErrorHandling: @pytest.mark.parametrize( "value",