Fix msgspec field ordering (#3292)

* Fix msgspec field ordering

* Address msgspec test failures

* Address msgspec review comments

* Update msgspec keyword-only expectations

* Cover msgspec test helpers

* Avoid unused msgspec Meta import

* Use e2e golden tests for msgspec fixes

Author: koxudaxi

src/datamodel_code_generator/model/msgspec.py

@@ -69,11 +69,10 @@ def __str__(self) -> str:
     from datamodel_code_generator.reference import Reference
 
 
-def _has_field_assignment(field: DataModelFieldBase) -> bool:
-    return (
-        bool(field.field)
-        or field.use_default_with_required
-        or not (field.required or (field.represented_default == "None" and field.strip_default_none))
+def has_field_assignment(field: DataModelFieldBase) -> bool:
+    """Return whether a msgspec field renders with a default assignment."""
+    return field.use_default_with_required or not (
+        field.required or (field.represented_default == "None" and field.strip_default_none)
     )
 
 
@@ -118,6 +117,7 @@ class Struct(DataModel):
     BASE_CLASS_ALIAS: ClassVar[str] = "_Struct"
     DEFAULT_IMPORTS: ClassVar[tuple[Import, ...]] = ()
     SUPPORTS_DISCRIMINATOR: ClassVar[bool] = True
+    SUPPORTS_KW_ONLY: ClassVar[bool] = True
     CONFIG_MAPPING: ClassVar[dict[tuple[str, Any], tuple[str, Any] | None]] = {
         ("allow_mutation", False): ("frozen", True),
         ("extra_fields", "forbid"): ("forbid_unknown_fields", True),
@@ -149,7 +149,7 @@ def __init__(  # noqa: PLR0913
         """Initialize msgspec Struct with fields sorted by field assignment requirement."""
         super().__init__(
             reference=reference,
-            fields=sorted(fields, key=_has_field_assignment),
+            fields=sorted(fields, key=has_field_assignment),
             decorators=decorators,
             base_classes=base_classes,
             custom_base_class=custom_base_class,
@@ -265,8 +265,8 @@ class DataModelField(DataModelFieldBase):
         "lt",
         "le",
         "multiple_of",
-        # 'min_items', # not supported by msgspec
-        # 'max_items', # not supported by msgspec
+        "min_items",
+        "max_items",
         "min_length",
         "max_length",
         "pattern",
@@ -398,6 +398,11 @@ def _get_meta_string(self) -> str | None:
                 },
             }
 
+        if (min_items := data.pop("min_items", None)) is not None:
+            data["min_length"] = min_items
+        if (max_items := data.pop("max_items", None)) is not None:
+            data["max_length"] = max_items
+
         meta_arguments = sorted(f"{k}={v!r}" for k, v in data.items() if v is not None)
         return f"Meta({', '.join(meta_arguments)})" if meta_arguments else None
 
@@ -448,7 +453,7 @@ def needs_annotated_import(self) -> bool:
     @property
     def needs_meta_import(self) -> bool:
         """Check if this field requires the Meta import."""
-        return self._get_meta_string() is not None
+        return self.use_annotated and self._get_meta_string() is not None
 
     def _get_default_as_struct_model(self) -> str | None:
         """Convert default value to Struct model using msgspec convert."""

src/datamodel_code_generator/model/template/msgspec.jinja2

@@ -27,7 +27,7 @@ class {{ class_name }}:
     {%- else %}
     {{ field.name }}: {{ field.type_hint }}
     {%- endif %}
-    {%- if not field.field and (not field.required or field.use_default_with_required or field.data_type.is_optional or field.nullable)
+    {%- if not field.field and (not field.required or field.use_default_with_required)
             %} = {{ field.represented_default }}
     {%- endif -%}
     {%- endif %}

src/datamodel_code_generator/parser/base.py

@@ -2480,12 +2480,17 @@ def __fix_dataclass_field_ordering(self, models: list[DataModel]) -> None:
             if (inherited := self.__get_dataclass_inherited_info(model)) is None:
                 continue
             inherited_names, has_default = inherited
-            if not has_default or not any(self.__is_new_required_field(f, inherited_names) for f in model.fields):
+            field_has_assignment = Parser._get_field_assignment_checker(model)
+            if not has_default or not any(
+                self.__is_new_required_field(f, inherited_names, field_has_assignment) for f in model.fields
+            ):
                 continue
 
-            if self.target_python_version.has_kw_only_dataclass:
+            if isinstance(model, msgspec_model.Struct):
+                model.add_base_class_kwarg("kw_only", "True")
+            elif self.target_python_version.has_kw_only_dataclass:
                 for field in model.fields:
-                    if self.__is_new_required_field(field, inherited_names):
+                    if self.__is_new_required_field(field, inherited_names, field_has_assignment):
                         field.extras["kw_only"] = True
             else:  # pragma: no cover
                 warn(
@@ -2496,16 +2501,20 @@ def __fix_dataclass_field_ordering(self, models: list[DataModel]) -> None:
                     category=UserWarning,
                     stacklevel=2,
                 )
-            self.generation_store.set_fields(model, sorted(model.fields, key=dataclass_model.has_field_assignment))
+            self.generation_store.set_fields(model, sorted(model.fields, key=field_has_assignment))
 
     @classmethod
     def __get_dataclass_inherited_info(cls, model: DataModel) -> tuple[set[str], bool] | None:
         """Get inherited field names and whether any has default. Returns None if not applicable."""
         if not model.SUPPORTS_KW_ONLY:
             return None
-        if not model.base_classes or model.dataclass_arguments.get("kw_only"):
+        base_class_kw_only = None
+        if isinstance(model, msgspec_model.Struct):
+            base_class_kw_only = model.extra_template_data.get("base_class_kwargs", {}).get("kw_only")
+        if not model.base_classes or model.dataclass_arguments.get("kw_only") or base_class_kw_only in {True, "True"}:
             return None
 
+        field_has_assignment = cls._get_field_assignment_checker(model)
         inherited_names: set[str] = set()
         has_default = False
         for base in model.base_classes:
@@ -2515,23 +2524,30 @@ def __get_dataclass_inherited_info(cls, model: DataModel) -> tuple[set[str], boo
                 if not f.name or f.extras.get("init") is False:
                     continue  # pragma: no cover
                 inherited_names.add(f.name)
-                if dataclass_model.has_field_assignment(f):
+                if field_has_assignment(f):
                     has_default = True
 
         for f in model.fields:
             if f.name not in inherited_names or f.extras.get("init") is False:
                 continue
-            if dataclass_model.has_field_assignment(f):  # pragma: no branch
+            if field_has_assignment(f):  # pragma: no branch
                 has_default = True
         return (inherited_names, has_default) if inherited_names else None
 
-    def __is_new_required_field(self, field: DataModelFieldBase, inherited: set[str]) -> bool:  # noqa: PLR6301
+    @staticmethod
+    def _get_field_assignment_checker(model: DataModel) -> Callable[[DataModelFieldBase], bool]:
+        if isinstance(model, msgspec_model.Struct):
+            return msgspec_model.has_field_assignment
+        return dataclass_model.has_field_assignment
+
+    def __is_new_required_field(  # noqa: PLR6301
+        self,
+        field: DataModelFieldBase,
+        inherited: set[str],
+        field_has_assignment: Callable[[DataModelFieldBase], bool],
+    ) -> bool:
         """Check if field is a new required init field."""
-        return (
-            field.name not in inherited
-            and field.extras.get("init") is not False
-            and not dataclass_model.has_field_assignment(field)
-        )
+        return field.name not in inherited and field.extras.get("init") is not False and not field_has_assignment(field)
 
     def __remove_overridden_models(self, models: list[DataModel]) -> list[DataModel]:
         """Remove models that are being overridden by custom types (model-level only).

tests/data/expected/main/jsonschema/msgspec_array_length_constraints_use_annotated.py

@@ -0,0 +1,13 @@
+# generated by datamodel-codegen:
+#   filename:  msgspec_array_length_constraints.json
+#   timestamp: 2019-07-26T00:00:00+00:00
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from msgspec import UNSET, Meta, Struct, UnsetType
+
+
+class Model(Struct):
+    items: Annotated[list[str], Meta(max_length=5, min_length=1)] | UnsetType = UNSET

tests/data/expected/main/jsonschema/msgspec_array_length_constraints_without_annotated.py

@@ -0,0 +1,11 @@
+# generated by datamodel-codegen:
+#   filename:  msgspec_array_length_constraints.json
+#   timestamp: 2019-07-26T00:00:00+00:00
+
+from __future__ import annotations
+
+from msgspec import UNSET, Struct, UnsetType
+
+
+class Model(Struct):
+    items: list[str] | UnsetType = UNSET

tests/data/expected/main/jsonschema/msgspec_inherited_optional_default.py

@@ -0,0 +1,20 @@
+# generated by datamodel-codegen:
+#   filename:  msgspec_inherited_optional_default.json
+#   timestamp: 2019-07-26T00:00:00+00:00
+
+from __future__ import annotations
+
+from msgspec import UNSET, Struct, UnsetType
+
+
+class Base(Struct):
+    kind: str | UnsetType = 'base'
+    note: str | UnsetType = UNSET
+
+
+class Child(Base, kw_only=True):
+    id: int
+
+
+class Model(Struct):
+    child: Child | UnsetType = UNSET

tests/data/expected/main/jsonschema/msgspec_required_alias_field.py

@@ -0,0 +1,12 @@
+# generated by datamodel-codegen:
+#   filename:  msgspec_required_alias_field.json
+#   timestamp: 2019-07-26T00:00:00+00:00
+
+from __future__ import annotations
+
+from msgspec import UNSET, Struct, UnsetType, field
+
+
+class Record(Struct):
+    req_id: int = field(name='req-id')
+    opt: str | UnsetType = UNSET

tests/data/expected/main/jsonschema/msgspec_required_nullable_field.py

@@ -0,0 +1,12 @@
+# generated by datamodel-codegen:
+#   filename:  msgspec_required_nullable_field.json
+#   timestamp: 2019-07-26T00:00:00+00:00
+
+from __future__ import annotations
+
+from msgspec import Struct
+
+
+class Record(Struct):
+    label: str | None
+    count: int

tests/data/expected/main/openapi/msgspec_use_annotated_with_field_constraints.py

@@ -15,7 +15,7 @@ class Pet(Struct):
     tag: Annotated[str, Meta(max_length=64)] | UnsetType = UNSET
 
 
-Pets: TypeAlias = list[Pet]
+Pets: TypeAlias = Annotated[list[Pet], Meta(max_length=10, min_length=1)]
 
 
 UID: TypeAlias = Annotated[int, Meta(ge=0)]
@@ -32,7 +32,7 @@ class User(Struct):
     name: Annotated[str, Meta(max_length=256)]
     uid: UID
     tag: Annotated[str, Meta(max_length=64)] | UnsetType = UNSET
-    phones: list[Phone] | UnsetType = UNSET
+    phones: Annotated[list[Phone], Meta(max_length=10)] | UnsetType = UNSET
     fax: list[FaxItem] | UnsetType = UNSET
     height: Annotated[int | float, Meta(ge=1.0, le=300.0)] | UnsetType = UNSET
     weight: Annotated[float | int, Meta(ge=1.0, le=1000.0)] | UnsetType = UNSET

tests/data/jsonschema/msgspec_array_length_constraints.json

@@ -0,0 +1,11 @@
+{
+    "type": "object",
+    "properties": {
+        "items": {
+            "type": "array",
+            "items": {"type": "string"},
+            "minItems": 1,
+            "maxItems": 5
+        }
+    }
+}