Fix duplicate file issue on every save/submit

Author: sandeepsajan0

hypha/apply/funds/models/mixins.py

@@ -1,3 +1,5 @@
+import hashlib
+
 from django.core.files import File
 from django.utils.safestring import mark_safe
 from django_file_form.models import PlaceholderUploadedFile
@@ -11,6 +13,7 @@
     MultiInputCharFieldBlock,
     UploadableMediaBlock,
 )
+from hypha.apply.stream_forms.files import StreamFieldFile
 from hypha.apply.utils.blocks import SingleIncludeMixin
 from hypha.apply.utils.storage import PrivateStorage
 
@@ -92,19 +95,87 @@ def process_file(cls, instance, field, file):
         else:
             return cls.stream_file(instance, field, file)
 
-    def process_file_data(self, data):
+    def process_file_data(self, data, latest_existing_data=None):
         for field in self.form_fields:
             if isinstance(field.block, UploadableMediaBlock):
-                file = self.process_file(self, field, data.get(field.id, []))
-                try:
-                    file.save()
-                except (AttributeError, FileNotFoundError):
+                new_file = data.get(field.id, [])
+                existing_file = latest_existing_data.get(field.id, [])
+
+                # processing files before checking because placeholder files can't be read
+                new_stream_file = self.process_file(self, field, new_file)
+                # existing_stream_file = self.process_file(self, field, existing_file)
+
+                # save only if it is not the same file(s)
+                same_file = self._is_same_file(existing_file, new_stream_file)
+                if not same_file:
                     try:
-                        for f in file:
-                            f.save()
-                    except FileNotFoundError:
-                        pass
-                self.form_data[field.id] = file
+                        new_stream_file.save()
+                    except (AttributeError, FileNotFoundError):
+                        try:
+                            for f in new_stream_file:
+                                f.save()
+                        except FileNotFoundError:
+                            pass
+                    self.form_data[field.id] = new_stream_file
+                else:
+                    self.form_data[field.id] = existing_file
+
+    def _is_same_file(self, existing, new):
+        # Normalize to list for multi-file support
+        if not isinstance(existing, list):
+            existing = [existing]
+        if not isinstance(new, list):
+            new = [new]
+
+        if len(existing) != len(new):
+            return False
+
+        for e, n in zip(existing, new, strict=False):
+            e_file = self._get_file_obj(e)
+            n_file = self._get_file_obj(n)
+            if not e_file or not n_file:
+                return False
+
+            # Compare file names
+            if e_file.name != n_file.name:
+                return False
+
+            # Compare file sizes
+            if e_file.size != n_file.size:
+                return False
+
+            # Compare file hashes(keep it after other checks to avoid checking it for every file)
+            if self._hash_file(e_file) != self._hash_file(n_file):
+                return False
+
+        return True
+
+    def _get_file_obj(self, file_obj):
+        """Returns a file-like object from Wagtail or Django file field, or None."""
+        try:
+            if isinstance(file_obj, StreamFieldFile):
+                return file_obj.file
+            if hasattr(file_obj, "file"):
+                return file_obj.file
+            elif hasattr(file_obj, "temporary_file_path"):
+                return open(file_obj.temporary_file_path(), "rb")
+            elif hasattr(file_obj, "read"):
+                return file_obj
+        except Exception:
+            pass
+        return None
+
+    def _hash_file(self, file_obj, chunk_size=4096):
+        """Returns SHA256 hash of a file-like object"""
+        try:
+            file_obj.seek(0)
+            hash_obj = hashlib.sha256()
+            while chunk := file_obj.read(chunk_size):
+                hash_obj.update(chunk)
+            file_obj.seek(0)
+            return hash_obj.hexdigest()
+        except Exception:
+            return None
 
     def extract_files(self):
         files = {}

hypha/apply/funds/models/submissions.py

@@ -743,7 +743,9 @@ def create_revision(
     def clean_submission(self):
         self.process_form_data()
         self.ensure_user_has_account()
-        self.process_file_data(self.form_data)
+        # pass current submission data to avoid file save on every submit(if file is not updated)
+        current_submission = ApplicationSubmission.objects.get(id=self.id)
+        self.process_file_data(self.form_data, current_submission.from_draft().raw_data)
 
     def get_assigned_meta_terms(self):
         """Returns assigned meta terms excluding the 'root' term"""
@@ -795,7 +797,7 @@ def save(self, *args, update_fields=None, skip_custom=False, **kwargs):
                     f"{self.get_from_parent('submission_id_prefix')}{self.id}"
                 )
 
-            self.process_file_data(files)
+            self.process_file_data(files, self.from_draft().raw_data)
             AssignedReviewers.objects.bulk_create_reviewers(
                 list(self.get_from_parent("reviewers").all()),
                 self,