Fix pydantic typed extra runtime compatibility

Author: koxudaxi

.github/workflows/test.yaml

@@ -97,6 +97,9 @@ jobs:
           - tox_env: pydantic25
             name: pydantic25
             coverage_name: pydantic25
+          - tox_env: pydantic213
+            name: pydantic213
+            coverage_name: pydantic213
     runs-on: ubuntu-24.04
     env:
       OS: ubuntu-24.04

src/datamodel_code_generator/parser/base.py

@@ -1426,10 +1426,7 @@ def __init__(  # noqa: PLR0912, PLR0915
         self.extra_template_data: defaultdict[str, Any] = config.extra_template_data or defaultdict(dict)
         self.validators = config.validators
         self.generate_schema_validators: bool = config.generate_schema_validators
-        if typed_extra_plain_annotation_key := self.data_model_type.TYPED_EXTRA_PLAIN_ANNOTATION_TEMPLATE_DATA_KEY:
-            self.extra_template_data.setdefault(ALL_MODEL, {})[typed_extra_plain_annotation_key] = (
-                config.target_python_version.has_native_deferred_annotations
-            )
+        self._set_typed_extra_annotation_mode(use_deferred_annotations=True)
 
         if self.validators:
             for model_name, model_config in self.validators.items():
@@ -2988,19 +2985,20 @@ def __update_type_aliases(
             )
             # When annotations are deferred (from __future__ or native PEP-649) only
             # TypeAliasBase / RootModel need quoting; regular DataModels are fine as-is.
-            # Class-body __annotations__ dicts are an exception: their right hand side is
-            # evaluated immediately, so typed extra references still need forward refs.
+            # Typed extra annotations are an exception: class-body __annotations__ dicts
+            # are evaluated immediately, and Pydantic can force native deferred annotations
+            # while constructing the class, so their forward references must stay quoted.
             process_all_fields = is_type_alias_or_root or not use_deferred_annotations
             if not process_all_fields and not any(
-                getattr(field, "use_pydantic_extra_annotations_dict", False) for field in model.fields
+                getattr(field, "is_pydantic_extra_field", False) for field in model.fields
             ):
                 continue
             if isinstance(model, TypeStatement):
                 continue
 
-            has_forward_ref = False
+            has_aliased_forward_ref = False
             for field in model.fields:
-                if not process_all_fields and not getattr(field, "use_pydantic_extra_annotations_dict", False):
+                if not process_all_fields and not getattr(field, "is_pydantic_extra_field", False):
                     continue
                 for data_type in field.data_type.all_data_types:
                     if not data_type.reference:
@@ -3017,10 +3015,10 @@ def __update_type_aliases(
                     if source_index is not None and source_index >= i:
                         data_type.alias = f'"{name}"'
                         cls.__disable_union_operator_for_forward_ref_parents(data_type)
-                        has_forward_ref = True
+                        has_aliased_forward_ref = True
 
-            if has_forward_ref:
-                model.has_forward_reference = True
+            if has_aliased_forward_ref:
+                model.has_forward_reference = model.has_forward_reference or process_all_fields
                 _clear_model_imports_cache_if_retained(model, can_retain_cache=can_retain_cache)
 
     @classmethod
@@ -3648,6 +3646,32 @@ def __get_resolve_reference_action_parts(
             ),
         ]
 
+    def _uses_deferred_annotations(
+        self,
+        with_import: bool | None,  # noqa: FBT001
+        disable_future_imports: bool,  # noqa: FBT001
+    ) -> bool:
+        """Return whether generated annotations use deferred evaluation."""
+        return bool(
+            self.target_python_version.has_native_deferred_annotations or (with_import and not disable_future_imports)
+        )
+
+    def _set_typed_extra_annotation_mode(self, *, use_deferred_annotations: bool) -> None:
+        """Select the safe typed-extra annotation form for the generated runtime."""
+        if not (key := self.data_model_type.TYPED_EXTRA_PLAIN_ANNOTATION_TEMPLATE_DATA_KEY):
+            return
+
+        native_deferred_annotations = self.target_python_version.has_native_deferred_annotations
+        match native_deferred_annotations:
+            case True:
+                use_plain_annotation = True
+            case False if use_deferred_annotations:
+                use_plain_annotation = False
+            case _:
+                use_plain_annotation = True
+
+        self.extra_template_data.setdefault(ALL_MODEL, {})[key] = use_plain_annotation
+
     def _prepare_parse_config(
         self,
         with_import: bool | None,  # noqa: FBT001
@@ -3657,9 +3681,7 @@ def _prepare_parse_config(
         module_split_mode: ModuleSplitMode | None,
     ) -> ParseConfig:
         """Prepare configuration for the parse operation."""
-        use_deferred_annotations = bool(
-            self.target_python_version.has_native_deferred_annotations or (with_import and not disable_future_imports)
-        )
+        use_deferred_annotations = self._uses_deferred_annotations(with_import, disable_future_imports)
 
         if (
             with_import
@@ -4058,6 +4080,9 @@ def parse(  # noqa: PLR0912, PLR0913, PLR0914, PLR0917
         collect_model_metadata: bool = False,  # noqa: FBT001, FBT002
     ) -> str | dict[tuple[str, ...], Result]:
         """Parse schema and generate code, returning single file or module dict."""
+        self._set_typed_extra_annotation_mode(
+            use_deferred_annotations=self._uses_deferred_annotations(with_import, disable_future_imports)
+        )
         self.parse_raw()
 
         config = self._prepare_parse_config(

tests/data/expected/main/jsonschema/additional_properties_nullable_self_ref_union_operator.py

@@ -0,0 +1,22 @@
+# generated by datamodel-codegen:
+#   filename:  additional_properties_nullable_self_ref.json
+#   timestamp: 2019-07-26T00:00:00+00:00
+
+from __future__ import annotations
+
+from typing import Dict, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+
+class Node(BaseModel):
+    model_config = ConfigDict(
+        extra='allow',
+    )
+    __annotations__ = {
+        '__pydantic_extra__': Dict[str, Optional["Node"]],
+    }
+    name: str
+
+
+Node.model_rebuild()

tests/data/expected/main/jsonschema/additional_properties_schema_with_properties_py313_no_future_imports.py

@@ -0,0 +1,13 @@
+# generated by datamodel-codegen:
+#   filename:  additional_properties_schema_with_properties.json
+#   timestamp: 2019-07-26T00:00:00+00:00
+
+from pydantic import BaseModel, ConfigDict
+
+
+class KnownAndExtra(BaseModel):
+    model_config = ConfigDict(
+        extra='allow',
+    )
+    __pydantic_extra__: dict[str, int]
+    name: str

tests/data/expected/main/jsonschema/additional_properties_self_ref_py314.py

@@ -0,0 +1,16 @@
+# generated by datamodel-codegen:
+#   filename:  additional_properties_self_ref.json
+#   timestamp: 2019-07-26T00:00:00+00:00
+
+from pydantic import BaseModel, ConfigDict
+
+
+class Node(BaseModel):
+    model_config = ConfigDict(
+        extra='allow',
+    )
+    __pydantic_extra__: dict[str, "Node"]
+    name: str
+
+
+Node.model_rebuild()

tests/data/expected/main/jsonschema/additional_properties_self_ref_union_operator_force_optional.py

@@ -0,0 +1,22 @@
+# generated by datamodel-codegen:
+#   filename:  additional_properties_self_ref.json
+#   timestamp: 2019-07-26T00:00:00+00:00
+
+from __future__ import annotations
+
+from typing import Dict
+
+from pydantic import BaseModel, ConfigDict
+
+
+class Node(BaseModel):
+    model_config = ConfigDict(
+        extra='allow',
+    )
+    __annotations__ = {
+        '__pydantic_extra__': Dict[str, "Node"],
+    }
+    name: str | None = None
+
+
+Node.model_rebuild()

tests/data/jsonschema/additional_properties_nullable_self_ref.json

@@ -0,0 +1,23 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Node",
+  "type": "object",
+  "properties": {
+    "name": {
+      "type": "string"
+    }
+  },
+  "required": [
+    "name"
+  ],
+  "additionalProperties": {
+    "anyOf": [
+      {
+        "$ref": "#"
+      },
+      {
+        "type": "null"
+      }
+    ]
+  }
+}

tests/main/jsonschema/test_main_jsonschema.py

@@ -6397,6 +6397,122 @@ def test_main_jsonschema_additional_properties_self_ref(output_file: Path) -> No
     )
 
 
+@BLACK_PY314_SKIP
+def test_main_jsonschema_additional_properties_self_ref_target_python_314(output_file: Path) -> None:
+    """Test Python 3.14 typed extras keep self-references safe at runtime."""
+    run_main_and_assert(
+        input_path=JSON_SCHEMA_DATA_PATH / "additional_properties_self_ref.json",
+        output_path=output_file,
+        input_file_type="jsonschema",
+        assert_func=assert_file_content,
+        expected_file="additional_properties_self_ref_py314.py",
+        extra_args=[
+            "--output-model-type",
+            "pydantic_v2.BaseModel",
+            "--target-python-version",
+            "3.14",
+        ],
+    )
+    assert_generated_model_json_validation(
+        output_file,
+        module_name="additional_properties_self_ref_py314",
+        model_name="Node",
+        valid_json='{"name":"root","child":{"name":"leaf"}}',
+        invalid_json='{"name":"root","child":{"name":"leaf","bad":1}}',
+        expected_error_type="model_type",
+        expected_attribute_path=("__pydantic_extra__", "child", "name"),
+        expected_attribute_value="leaf",
+    )
+
+
+def test_main_jsonschema_additional_properties_scalar_no_future_imports_target_python_313(
+    output_file: Path,
+) -> None:
+    """Test typed extras do not hide ordinary fields without deferred annotations."""
+    run_main_and_assert(
+        input_path=JSON_SCHEMA_DATA_PATH / "additional_properties_schema_with_properties.json",
+        output_path=output_file,
+        input_file_type="jsonschema",
+        assert_func=assert_file_content,
+        expected_file="additional_properties_schema_with_properties_py313_no_future_imports.py",
+        extra_args=[
+            "--output-model-type",
+            "pydantic_v2.BaseModel",
+            "--target-python-version",
+            "3.13",
+            "--disable-future-imports",
+        ],
+        force_exec_validation=True,
+    )
+    assert_generated_model_json_validation(
+        output_file,
+        module_name="additional_properties_schema_with_properties_py313_no_future_imports",
+        model_name="KnownAndExtra",
+        valid_json='{"name":"known","size":1}',
+        invalid_json='{"name":"known","size":[]}',
+        expected_error_type="int_type",
+        expected_attribute_path=("__pydantic_extra__",),
+        expected_attribute_value={"size": 1},
+    )
+
+
+def test_main_jsonschema_additional_properties_self_ref_use_union_operator_force_optional(
+    output_file: Path,
+) -> None:
+    """Test typed self-references do not rewrite unrelated PEP 604 annotations."""
+    run_main_and_assert(
+        input_path=JSON_SCHEMA_DATA_PATH / "additional_properties_self_ref.json",
+        output_path=output_file,
+        input_file_type="jsonschema",
+        assert_func=assert_file_content,
+        expected_file="additional_properties_self_ref_union_operator_force_optional.py",
+        extra_args=[
+            "--output-model-type",
+            "pydantic_v2.BaseModel",
+            "--use-union-operator",
+            "--force-optional",
+        ],
+        force_exec_validation=True,
+    )
+    assert_generated_model_json_validation(
+        output_file,
+        module_name="additional_properties_self_ref_union_operator_force_optional",
+        model_name="Node",
+        valid_json='{"name":"root","child":{"name":"leaf"}}',
+        invalid_json='{"name":"root","child":{"name":"leaf","bad":1}}',
+        expected_error_type="model_type",
+        expected_attribute_path=("__pydantic_extra__", "child", "name"),
+        expected_attribute_value="leaf",
+    )
+
+
+def test_main_jsonschema_additional_properties_nullable_self_ref_use_union_operator(output_file: Path) -> None:
+    """Test typed-extra forward refs invalidate retained import caches."""
+    run_main_and_assert(
+        input_path=JSON_SCHEMA_DATA_PATH / "additional_properties_nullable_self_ref.json",
+        output_path=output_file,
+        input_file_type="jsonschema",
+        assert_func=assert_file_content,
+        expected_file="additional_properties_nullable_self_ref_union_operator.py",
+        extra_args=[
+            "--output-model-type",
+            "pydantic_v2.BaseModel",
+            "--use-union-operator",
+        ],
+        force_exec_validation=True,
+    )
+    assert_generated_model_json_validation(
+        output_file,
+        module_name="additional_properties_nullable_self_ref_union_operator",
+        model_name="Node",
+        valid_json='{"name":"root","child":{"name":"leaf"},"empty":null}',
+        invalid_json='{"name":"root","child":{"name":"leaf","bad":1}}',
+        expected_error_type="model_type",
+        expected_attribute_path=("__pydantic_extra__", "child", "name"),
+        expected_attribute_value="leaf",
+    )
+
+
 def test_main_jsonschema_property_names_type_non_string(output_file: Path) -> None:
     """Test non-string propertyNames type rejects every JSON object key."""
     run_main_and_assert(