reimport: optimize vulnerability_id processing

Author: valentijnscholten

dojo/finding/deduplication.py

@@ -239,7 +239,7 @@ def build_dedupe_scope_queryset(test):
     return (
         Finding.objects.filter(scope_q)
         .select_related("test", "test__engagement", "test__test_type")
-        .prefetch_related("endpoints")
+        .prefetch_related("endpoints", "vulnerability_id_set")
     )
 
 

dojo/importers/base_importer.py

@@ -793,9 +793,34 @@ def process_vulnerability_ids(
     ) -> Finding:
         """
         Parse the `unsaved_vulnerability_ids` field from findings after they are parsed
-        to create `Vulnerability_Id` objects with the finding associated correctly
+        to create `Vulnerability_Id` objects with the finding associated correctly.
+        Only updates if vulnerability_ids have changed to avoid unnecessary database operations.
+
+        Args:
+            finding: The finding to process vulnerability IDs for
+
+        Returns:
+            The finding object
+
         """
         if finding.unsaved_vulnerability_ids:
+            # Only check for changes if the finding has been saved and might have existing vulnerability_ids
+            # For new findings, we always need to save vulnerability_ids
+            if finding.pk and finding.vulnerability_id_set.exists():
+                # Check if vulnerability_ids have changed before updating
+                # Get existing vulnerability IDs from the database
+                existing_vuln_ids = set(finding.vulnerability_ids) if finding.vulnerability_ids else set()
+                # Normalize the new vulnerability IDs (remove duplicates for comparison)
+                new_vuln_ids = set(finding.unsaved_vulnerability_ids)
+
+                # Only update if vulnerability IDs have changed
+                if existing_vuln_ids == new_vuln_ids:
+                    logger.debug(
+                        f"Skipping vulnerability_ids update for finding {finding.id} - "
+                        f"vulnerability_ids unchanged: {sorted(existing_vuln_ids)}",
+                    )
+                    return finding
+
             # Remove old vulnerability ids - keeping this call only because of flake8
             Vulnerability_Id.objects.filter(finding=finding).delete()
 

unittests/test_importers_performance.py

@@ -217,11 +217,11 @@ def test_import_reimport_reimport_performance_no_async(self):
         testuser.usercontactinfo.block_execution = True
         testuser.usercontactinfo.save()
         self._import_reimport_performance(
-            expected_num_queries1=345,
+            expected_num_queries1=346,
             expected_num_async_tasks1=6,
-            expected_num_queries2=293,
+            expected_num_queries2=294,
             expected_num_async_tasks2=17,
-            expected_num_queries3=180,
+            expected_num_queries3=181,
             expected_num_async_tasks3=16,
         )
 
@@ -239,11 +239,11 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self):
         testuser.usercontactinfo.save()
 
         self._import_reimport_performance(
-            expected_num_queries1=311,
+            expected_num_queries1=312,
             expected_num_async_tasks1=6,
-            expected_num_queries2=286,
+            expected_num_queries2=287,
             expected_num_async_tasks2=17,
-            expected_num_queries3=175,
+            expected_num_queries3=176,
             expected_num_async_tasks3=16,
         )
 
@@ -265,11 +265,11 @@ def test_import_reimport_reimport_performance_no_async_with_product_grading(self
         self.system_settings(enable_product_grade=True)
 
         self._import_reimport_performance(
-            expected_num_queries1=347,
+            expected_num_queries1=348,
             expected_num_async_tasks1=8,
-            expected_num_queries2=295,
+            expected_num_queries2=296,
             expected_num_async_tasks2=19,
-            expected_num_queries3=182,
+            expected_num_queries3=183,
             expected_num_async_tasks3=18,
         )
 
@@ -288,11 +288,11 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr
         self.system_settings(enable_product_grade=True)
 
         self._import_reimport_performance(
-            expected_num_queries1=313,
+            expected_num_queries1=314,
             expected_num_async_tasks1=8,
-            expected_num_queries2=288,
+            expected_num_queries2=289,
             expected_num_async_tasks2=19,
-            expected_num_queries3=177,
+            expected_num_queries3=178,
             expected_num_async_tasks3=18,
         )
 
@@ -449,9 +449,9 @@ def test_deduplication_performance_no_async(self):
         testuser.usercontactinfo.save()
 
         self._deduplication_performance(
-            expected_num_queries1=316,
+            expected_num_queries1=317,
             expected_num_async_tasks1=7,
-            expected_num_queries2=287,
+            expected_num_queries2=288,
             expected_num_async_tasks2=7,
         )
 
@@ -469,8 +469,8 @@ def test_deduplication_performance_pghistory_no_async(self):
         testuser.usercontactinfo.save()
 
         self._deduplication_performance(
-            expected_num_queries1=280,
+            expected_num_queries1=281,
             expected_num_async_tasks1=7,
-            expected_num_queries2=250,
+            expected_num_queries2=251,
             expected_num_async_tasks2=7,
         )