perf(importers): batch BurpRawRequestResponse inserts + re-enable perf tests (#14969)

* perf(importers): batch BurpRawRequestResponse inserts + re-enable perf tests

Replace per-finding save() calls in process_request_response_pairs with
bulk_create at batch boundaries, mirroring the location_handler pattern.
Reduces DB round-trips proportionally to findings with req/resp data.
Drops the no-op clean() calls (BurpRawRequestResponse has no custom clean).

Re-enable TestDojoImporterPerformanceSmall and TestDojoImporterPerformanceSmallLocations
with recalibrated query counts after the RBAC→legacy authorization migration.

* test(perf): recalibrate tag inheritance ZAP query counts

Batch BurpRawRequestResponse inserts reduce per-finding saves for the ZAP
parser (which emits req/resp pairs). Update expected counts to match.

Author: valentijnscholten

dojo/importers/base_importer.py

@@ -80,6 +80,7 @@ def __init__(
         ImporterOptions.__init__(self, *args, **kwargs)
         self.pending_vulnerability_ids: list[Vulnerability_Id] = []
         self.pending_vuln_id_deletes: list[int] = []
+        self.pending_burp_rr: list[BurpRawRequestResponse] = []
 
     def check_child_implementation_exception(self):
         """
@@ -719,24 +720,26 @@ def process_request_response_pairs(
         Create BurpRawRequestResponse objects linked to the finding without
         returning the finding afterward
         """
-        if len(unsaved_req_resp := getattr(finding, "unsaved_req_resp", [])) > 0:
-            for req_resp in unsaved_req_resp:
-                burp_rr = BurpRawRequestResponse(
-                    finding=finding,
-                    burpRequestBase64=base64.b64encode(req_resp["req"].encode("utf-8")),
-                    burpResponseBase64=base64.b64encode(req_resp["resp"].encode("utf-8")))
-                burp_rr.clean()
-                burp_rr.save()
+        for req_resp in getattr(finding, "unsaved_req_resp", []):
+            self.pending_burp_rr.append(BurpRawRequestResponse(
+                finding=finding,
+                burpRequestBase64=base64.b64encode(req_resp["req"].encode("utf-8")),
+                burpResponseBase64=base64.b64encode(req_resp["resp"].encode("utf-8")),
+            ))
 
         unsaved_request = getattr(finding, "unsaved_request", None)
         unsaved_response = getattr(finding, "unsaved_response", None)
         if unsaved_request is not None and unsaved_response is not None:
-            burp_rr = BurpRawRequestResponse(
+            self.pending_burp_rr.append(BurpRawRequestResponse(
                 finding=finding,
                 burpRequestBase64=base64.b64encode(unsaved_request.encode()),
-                burpResponseBase64=base64.b64encode(unsaved_response.encode()))
-            burp_rr.clean()
-            burp_rr.save()
+                burpResponseBase64=base64.b64encode(unsaved_response.encode()),
+            ))
+
+    def flush_burp_request_response(self) -> None:
+        if self.pending_burp_rr:
+            BurpRawRequestResponse.objects.bulk_create(self.pending_burp_rr, batch_size=1000)
+            self.pending_burp_rr.clear()
 
     def process_locations(
         self,

dojo/importers/default_importer.py

@@ -276,6 +276,7 @@ def _process_findings_internal(
             if len(batch_finding_ids) >= batch_max_size or is_final_finding:
                 self.location_handler.persist()
                 self.flush_vulnerability_ids()
+                self.flush_burp_request_response()
                 # Apply parser-supplied tags for this batch before post-processing starts,
                 # so rules/deduplication tasks see the tags already on the findings.
                 bulk_apply_parser_tags(findings_with_parser_tags)

dojo/importers/default_reimporter.py

@@ -440,6 +440,7 @@ def _process_findings_internal(
                     if len(batch_finding_ids) >= dedupe_batch_max_size or is_final:
                         self.location_handler.persist()
                         self.flush_vulnerability_ids()
+                        self.flush_burp_request_response()
                         # Apply parser-supplied tags for this batch before post-processing starts,
                         # so rules/deduplication tasks see the tags already on the findings.
                         bulk_apply_parser_tags(findings_with_parser_tags)
@@ -564,6 +565,7 @@ def close_old_findings(
         # Persist any accumulated location/endpoint status changes
         self.location_handler.persist()
         self.flush_vulnerability_ids()
+        self.flush_burp_request_response()
         # push finding groups to jira since we only only want to push whole groups
         # We dont check if the finding jira sync is applicable quite yet until we can get in the loop
         # but this is a way to at least make it that far

unittests/test_tag_inheritance_perf.py

@@ -590,9 +590,9 @@ def test_baseline_zap_scan_reimport_with_new_findings_v3(self):
     # the async watson indexer, executed inline under CELERY_TASK_ALWAYS_EAGER);
     # +5 reimport (no-change + with-new) queries from removal of
     # WATSON_ASYNC_INDEX_UPDATE_THRESHOLD making async dispatch unconditional.
-    EXPECTED_ZAP_IMPORT_V2 = 368
-    EXPECTED_ZAP_IMPORT_V3 = 392
+    EXPECTED_ZAP_IMPORT_V2 = 287
+    EXPECTED_ZAP_IMPORT_V3 = 311
     EXPECTED_ZAP_REIMPORT_NO_CHANGE_V2 = 74
     EXPECTED_ZAP_REIMPORT_NO_CHANGE_V3 = 86
-    EXPECTED_ZAP_REIMPORT_WITH_NEW_V2 = 170
-    EXPECTED_ZAP_REIMPORT_WITH_NEW_V3 = 199
+    EXPECTED_ZAP_REIMPORT_WITH_NEW_V2 = 148
+    EXPECTED_ZAP_REIMPORT_WITH_NEW_V3 = 177