Fix partial form data updates involving ListField (#9902)

* respecting ordered sequence while partial update is fixed

* getting field order is re-ordered for backward compatibility

* more detailed tests are added

* .idea is added for Pycharm tmp files

* reduntant check is removed, respecting o(n) search, checking prefixed keys is improved

* missing tests are added, more detailed

* Update tests/test_serializer_lists.py

* comment is fixed for current solution explanation

* standardized the errors

* Update rest_framework/fields.py

* listfield html input tests are seperated

* unnecessary try/except is removed, explanation parts are refactored

---------

Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} <auvipy@gmail.com>

Author: Natgho

.gitignore

@@ -19,3 +19,5 @@ coverage.*
 !.github
 !.gitignore
 !.pre-commit-config.yaml
+
+.idea

rest_framework/fields.py

@@ -1680,18 +1680,24 @@ def __init__(self, **kwargs):
             self.validators.append(MinLengthValidator(self.min_length, message=message))
 
     def get_value(self, dictionary):
-        if self.field_name not in dictionary:
-            if getattr(self.root, 'partial', False):
-                return empty
         # We override the default field access in order to support
         # lists in HTML forms.
         if html.is_html_input(dictionary):
             val = dictionary.getlist(self.field_name, [])
             if len(val) > 0:
-                # Support QueryDict lists in HTML input.
+                # Support QueryDict lists and other list-like results in HTML input.
                 return val
+            # For partial updates, avoid calling parse_html_list unless indexed keys are present.
+            # This reduces unnecessary parsing overhead for omitted list fields.
+            if getattr(self.root, 'partial', False):
+                prefix = self.field_name + '['
+                if not any(key.startswith(prefix) for key in dictionary):
+                    return empty
             return html.parse_html_list(dictionary, prefix=self.field_name, default=empty)
 
+        # Non-HTML input: standard dictionary access
+        if self.field_name not in dictionary and getattr(self.root, 'partial', False):
+            return empty
         return dictionary.get(self.field_name, empty)
 
     def to_internal_value(self, data):

tests/test_fields.py

@@ -576,6 +576,67 @@ class TestSerializer(serializers.Serializer):
         assert serializer.is_valid()
         assert serializer.validated_data == {'scores': ['']}
 
+    def test_partial_update_with_indexed_keys(self):
+        """
+        Regression test for indexed HTML form keys with partial=True.
+        When data is passed as `colors[0]=#ffffff&colors[1]=#000000`
+        with partial=True, the field should parse indexed keys correctly.
+        """
+        class TestSerializer(serializers.Serializer):
+            colors = serializers.ListField(
+                allow_null=True,
+                child=serializers.CharField(max_length=7),
+                required=False
+            )
+            name = serializers.CharField(max_length=100, required=False)
+
+        serializer = TestSerializer(
+            data=QueryDict('colors[0]=#ffffff&colors[1]=#000000'),
+            partial=True
+        )
+        assert serializer.is_valid()
+        assert serializer.validated_data == {'colors': ['#ffffff', '#000000']}
+
+    def test_partial_update_omitted_list_field(self):
+        """
+        When a ListField is omitted in a partial update (and there are no
+        indexed keys for it), the field should be skipped and not included in
+        the validated data.
+        """
+        class TestSerializer(serializers.Serializer):
+            colors = serializers.ListField(
+                child=serializers.CharField(max_length=7),
+                required=False
+            )
+            name = serializers.CharField(max_length=100)
+
+        # colors is omitted, only name is provided
+        serializer = TestSerializer(
+            data=QueryDict('name=Test'),
+            partial=True
+        )
+        assert serializer.is_valid()
+        assert serializer.validated_data == {'name': 'Test'}
+        assert 'colors' not in serializer.validated_data
+
+    def test_partial_update_indexed_keys_ordering(self):
+        """
+        Indexed keys should preserve the correct order even when
+        they appear out of order in the QueryDict.
+        """
+        class TestSerializer(serializers.Serializer):
+            items = serializers.ListField(
+                child=serializers.IntegerField(),
+                required=False
+            )
+
+        serializer = TestSerializer(
+            data=QueryDict('items[2]=3&items[0]=1&items[1]=2'),
+            partial=True
+        )
+        assert serializer.is_valid()
+        assert serializer.validated_data == {'items': [1, 2, 3]}
+
 
 class TestCreateOnlyDefault:
     def setup_method(self):

tests/test_serializer_lists.py

@@ -286,6 +286,51 @@ def test_validate_html_input(self):
         assert serializer.validated_data == expected_output
 
 
+class TestListFieldHTMLInput:
+    """
+    Tests for ListField with HTML form input, including indexed keys.
+    """
+
+    def test_listfield_with_indexed_keys(self):
+        """
+        Test that indexed keys (e.g., field[0], field[1]) work correctly
+        in HTML form submissions.
+        """
+        class CommunitySerializer(serializers.Serializer):
+            colors = serializers.ListField(
+                allow_null=True,
+                child=serializers.CharField(label='Colors', max_length=7),
+                required=False
+            )
+        # Simulate form data with indexed keys
+        data = MultiValueDict({
+            'colors[0]': ['#ffffff'],
+            'colors[1]': ['#000000']
+        })
+        serializer = CommunitySerializer(data=data)
+        assert serializer.is_valid()
+        assert 'colors' in serializer.validated_data
+        assert serializer.validated_data['colors'] == ['#ffffff', '#000000']
+
+    def test_listfield_standard_form_submission(self):
+        """
+        Test standard HTML form list submission (e.g., multi-select).
+        Ensures backward compatibility with existing behavior.
+        """
+        class CommunitySerializer(serializers.Serializer):
+            colors = serializers.ListField(
+                child=serializers.CharField(label='Colors', max_length=7),
+                required=True
+            )
+        # Standard multi-select form submission
+        data = MultiValueDict({
+            'colors': ['#ffffff', '#000000', '#ff0000']
+        })
+        serializer = CommunitySerializer(data=data)
+        assert serializer.is_valid()
+        assert serializer.validated_data['colors'] == ['#ffffff', '#000000', '#ff0000']
+
+
 class TestNestedListSerializerAllowEmpty:
     """Tests the behavior of allow_empty=False when a ListSerializer is used as a field."""
 
@@ -426,6 +471,69 @@ class MultipleChoiceSerializer(serializers.Serializer):
         assert serializer.validated_data == {}
         assert serializer.errors == {}
 
+    def test_partial_listfield_with_non_indexed_list(self):
+        """
+        Test that ListField still works with non-indexed list submission
+        in partial updates (backward compatibility check).
+        """
+        class CommunitySerializer(serializers.Serializer):
+            colors = serializers.ListField(
+                allow_null=True,
+                child=serializers.CharField(label='Colors', max_length=7),
+                required=False
+            )
+        # Simulate standard HTML form list (e.g., multiple select)
+        data = MultiValueDict({
+            'colors': ['#ffffff', '#000000']
+        })
+        serializer = CommunitySerializer(data=data, partial=True)
+        assert serializer.is_valid()
+        assert 'colors' in serializer.validated_data
+        assert serializer.validated_data['colors'] == ['#ffffff', '#000000']
+
+    def test_listfield_mixed_plain_and_indexed_keys(self):
+        """
+        Test that when both plain field and indexed keys are present,
+        the plain field takes precedence (standard HTML form behavior).
+        """
+        class CommunitySerializer(serializers.Serializer):
+            colors = serializers.ListField(
+                allow_null=True,
+                child=serializers.CharField(label='Colors', max_length=7),
+                required=False
+            )
+        # When both present, getlist should win (standard HTML form behavior)
+        data = MultiValueDict({
+            'colors': ['#aaaaaa', '#bbbbbb'],  # This should be used
+            'colors[0]': ['#ffffff'],           # These should be ignored
+            'colors[1]': ['#000000']
+        })
+        serializer = CommunitySerializer(data=data, partial=True)
+        assert serializer.is_valid()
+        assert 'colors' in serializer.validated_data
+        # Plain field values should take precedence
+        assert serializer.validated_data['colors'] == ['#aaaaaa', '#bbbbbb']
+
+    def test_partial_listfield_no_data_returns_empty(self):
+        """
+        Test that when a ListField is omitted in partial updates,
+        it does not appear in validated_data (not even as an empty list).
+        """
+        class CommunitySerializer(serializers.Serializer):
+            name = serializers.CharField(max_length=100)
+            colors = serializers.ListField(
+                allow_null=True,
+                child=serializers.CharField(label='Colors', max_length=7),
+                required=False
+            )
+        data = MultiValueDict({
+            'name': ['Community Name']
+        })
+        serializer = CommunitySerializer(data=data, partial=True)
+        assert serializer.is_valid()
+        assert 'name' in serializer.validated_data
+        assert 'colors' not in serializer.validated_data  # Should be skipped
+
     def test_allow_empty_true(self):
         class ListSerializer(serializers.Serializer):
             update_field = serializers.IntegerField()