fix(importers): dispatch post-processing with per-finding push_to_jira (#15186)

The per-batch post-processing dispatch read the push_to_jira flag
computed for the last finding of the batch and applied it to the whole
batch. With finding groups enabled and push_to_jira on, a mixed batch
ending in a grouped finding suppressed the JIRA push for every
ungrouped finding in the batch, and a batch ending in an ungrouped
finding pushed grouped findings individually.

Track each finding's own flag and partition the batch by flag at
dispatch time. Uniform batches (grouping disabled, push off) still
dispatch once, so task counts are unchanged for the common case.
Applies to both import and reimport.

Author: valentijnscholten

dojo/importers/default_importer.py

@@ -182,8 +182,12 @@ def _process_findings_internal(
         parsed_findings: list[Finding],
         **kwargs: dict,
     ) -> list[Finding]:
-        # Batched post-processing (no chord): dispatch a task per 1000 findings or on final finding
-        batch_finding_ids: list[int] = []
+        # Batched post-processing (no chord): dispatch a task per 1000 findings or on final finding.
+        # Each entry carries the finding's own push_to_jira flag: grouped findings are pushed to
+        # JIRA as a group after the loop, so their individual push is suppressed while ungrouped
+        # findings in the same batch must still be pushed. The batch is partitioned by flag at
+        # dispatch time instead of applying one finding's flag to the whole batch.
+        batch_finding_ids: list[tuple[int, bool]] = []
         batch_findings: list[Finding] = []
         batch_max_size = getattr(settings, "IMPORT_REIMPORT_DEDUPE_BATCH_SIZE", 1000)
 
@@ -274,7 +278,7 @@ def _process_findings_internal(
                          self.push_to_jira, self.findings_groups_enabled, self.group_by)
             push_to_jira = self.push_to_jira and ((not self.findings_groups_enabled or not self.group_by) or not finding_will_be_grouped)
             logger.debug("process_findings: computed push_to_jira=%s", push_to_jira)
-            batch_finding_ids.append(finding.id)
+            batch_finding_ids.append((finding.id, push_to_jira))
             batch_findings.append(finding)
 
             # If batch is full or we're at the end, persist locations/endpoints and dispatch
@@ -293,25 +297,31 @@ def _process_findings_internal(
                 # dispatches, so rules/dedup see inherited tags on .tags.
                 apply_inherited_tags_for_findings(batch_findings)
                 batch_findings.clear()
-                finding_ids_batch = list(batch_finding_ids)
+                # Partition the batch by each finding's own push_to_jira flag so one
+                # finding's grouping state is not applied to the whole batch. Uniform
+                # batches (grouping disabled, or push_to_jira off) stay a single dispatch.
+                finding_ids_by_push: dict[bool, list[int]] = {}
+                for finding_id, finding_push_to_jira in batch_finding_ids:
+                    finding_ids_by_push.setdefault(finding_push_to_jira, []).append(finding_id)
                 batch_finding_ids.clear()
-                logger.debug("process_findings: dispatching batch with push_to_jira=%s (batch_size=%d, is_final=%s)",
-                             push_to_jira, len(finding_ids_batch), is_final_finding)
-                result = dojo_dispatch_task(
-                    finding_helper.post_process_findings_batch,
-                    finding_ids_batch,
-                    dedupe_option=True,
-                    rules_option=True,
-                    product_grading_option=True,
-                    issue_updater_option=True,
-                    push_to_jira=push_to_jira,
-                    # 'async_wait' joins on this dispatch via AsyncResult.get(), so its
-                    # result must be stored despite the global CELERY_TASK_IGNORE_RESULT.
-                    **({"ignore_result": False} if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT else {}),
-                    **self.post_processing_dispatch_kwargs(**kwargs),
-                )
-                if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT:
-                    self.record_post_processing_result(result)
+                for push_to_jira_batch, finding_ids_batch in finding_ids_by_push.items():
+                    logger.debug("process_findings: dispatching batch with push_to_jira=%s (batch_size=%d, is_final=%s)",
+                                 push_to_jira_batch, len(finding_ids_batch), is_final_finding)
+                    result = dojo_dispatch_task(
+                        finding_helper.post_process_findings_batch,
+                        finding_ids_batch,
+                        dedupe_option=True,
+                        rules_option=True,
+                        product_grading_option=True,
+                        issue_updater_option=True,
+                        push_to_jira=push_to_jira_batch,
+                        # 'async_wait' joins on this dispatch via AsyncResult.get(), so its
+                        # result must be stored despite the global CELERY_TASK_IGNORE_RESULT.
+                        **({"ignore_result": False} if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT else {}),
+                        **self.post_processing_dispatch_kwargs(**kwargs),
+                    )
+                    if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT:
+                        self.record_post_processing_result(result)
 
             # No chord: tasks are dispatched immediately above per batch
 

dojo/importers/default_reimporter.py

@@ -320,7 +320,11 @@ def _process_findings_internal(
                 continue
             cleaned_findings.append(sanitized)
 
-        batch_finding_ids: list[int] = []
+        # Each entry carries the finding's own push_to_jira flag: grouped findings are pushed to
+        # JIRA as a group, so their individual push is suppressed while ungrouped findings in the
+        # same batch must still be pushed. The batch is partitioned by flag at dispatch time
+        # instead of applying one finding's flag to the whole batch.
+        batch_finding_ids: list[tuple[int, bool]] = []
         batch_findings: list[Finding] = []
         # Findings that were newly created (else branch below) — pass these to
         # `apply_inherited_tags_for_findings` instead of `batch_findings` so
@@ -423,7 +427,7 @@ def _process_findings_internal(
                     )
                     # all data is already saved on the finding, we only need to trigger post processing in batches
                     push_to_jira = self.push_to_jira and ((not self.findings_groups_enabled or not self.group_by) or not finding_will_be_grouped)
-                    batch_finding_ids.append(finding.id)
+                    batch_finding_ids.append((finding.id, push_to_jira))
                     batch_findings.append(finding)
 
                     # Post-processing batches (deduplication, rules, etc.) are separate from matching batches.
@@ -460,24 +464,30 @@ def _process_findings_internal(
                         apply_inherited_tags_for_findings(new_findings_in_batch)
                         new_findings_in_batch.clear()
                         batch_findings.clear()
-                        finding_ids_batch = list(batch_finding_ids)
+                        # Partition the batch by each finding's own push_to_jira flag so one
+                        # finding's grouping state is not applied to the whole batch. Uniform
+                        # batches (grouping disabled, or push_to_jira off) stay a single dispatch.
+                        finding_ids_by_push: dict[bool, list[int]] = {}
+                        for finding_id, finding_push_to_jira in batch_finding_ids:
+                            finding_ids_by_push.setdefault(finding_push_to_jira, []).append(finding_id)
                         batch_finding_ids.clear()
-                        result = dojo_dispatch_task(
-                            finding_helper.post_process_findings_batch,
-                            finding_ids_batch,
-                            dedupe_option=True,
-                            rules_option=True,
-                            product_grading_option=True,
-                            issue_updater_option=True,
-                            push_to_jira=push_to_jira,
-                            jira_instance_id=getattr(self.jira_instance, "id", None),
-                            # 'async_wait' joins on this dispatch via AsyncResult.get(), so its
-                            # result must be stored despite the global CELERY_TASK_IGNORE_RESULT.
-                            **({"ignore_result": False} if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT else {}),
-                            **self.post_processing_dispatch_kwargs(**kwargs),
-                        )
-                        if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT:
-                            self.record_post_processing_result(result)
+                        for push_to_jira_batch, finding_ids_batch in finding_ids_by_push.items():
+                            result = dojo_dispatch_task(
+                                finding_helper.post_process_findings_batch,
+                                finding_ids_batch,
+                                dedupe_option=True,
+                                rules_option=True,
+                                product_grading_option=True,
+                                issue_updater_option=True,
+                                push_to_jira=push_to_jira_batch,
+                                jira_instance_id=getattr(self.jira_instance, "id", None),
+                                # 'async_wait' joins on this dispatch via AsyncResult.get(), so its
+                                # result must be stored despite the global CELERY_TASK_IGNORE_RESULT.
+                                **({"ignore_result": False} if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT else {}),
+                                **self.post_processing_dispatch_kwargs(**kwargs),
+                            )
+                            if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT:
+                                self.record_post_processing_result(result)
 
         # No chord: tasks are dispatched immediately above per batch
 

unittests/test_importers_importer.py

@@ -1143,3 +1143,91 @@ def test_reactivation_of_non_duplicate_still_activates(self):
         self.assertIsNone(result_finding.mitigated)
         self.assertTrue(result_finding.active)
         self.assertTrue(result_finding.verified)
+
+
+# Regression: the per-batch post-processing dispatch read the push_to_jira flag computed
+# for the LAST finding of the batch and applied it to the entire batch. With finding
+# groups enabled and push_to_jira=True, a mixed batch ending in a grouped finding
+# suppressed the JIRA push for every ungrouped finding in the batch (and vice versa).
+class TestDojoImporterBatchPushToJira(DojoTestCase):
+    def _process_findings_with_groups(self, parsed_findings):
+        """Run process_findings with push_to_jira + group_by and capture per-finding dispatch flags."""
+        scan_type = "Acunetix Scan"
+        user, _ = User.objects.get_or_create(username="admin")
+        product_type, _ = Product_Type.objects.get_or_create(name="batch_push")
+        product, _ = Product.objects.get_or_create(
+            name="TestDojoImporterBatchPushToJira",
+            description="test product",
+            prod_type=product_type,
+        )
+        engagement, _ = Engagement.objects.get_or_create(
+            name="Batch Push To Jira",
+            product=product,
+            target_start=timezone.now(),
+            target_end=timezone.now(),
+        )
+        environment, _ = Development_Environment.objects.get_or_create(name="Development")
+        self.system_settings(enable_finding_groups=True)
+        importer = DefaultImporter(
+            user=user,
+            lead=user,
+            scan_date=None,
+            environment=environment,
+            minimum_severity="Info",
+            active=True,
+            verified=True,
+            scan_type=scan_type,
+            engagement=engagement,
+            push_to_jira=True,
+            group_by="component_name",
+        )
+        test = importer.create_test(scan_type)
+        for finding in parsed_findings:
+            finding.test = test
+        with (
+            patch("dojo.importers.default_importer.dojo_dispatch_task") as dispatch_mock,
+            patch("dojo.importers.default_importer.jira_services.push"),
+        ):
+            new_findings = importer.process_findings(parsed_findings)
+        # Map each dispatched finding id to the push_to_jira flag its batch was sent with
+        flag_by_finding_id = {}
+        for call in dispatch_mock.call_args_list:
+            finding_ids = call.args[1]
+            for finding_id in finding_ids:
+                flag_by_finding_id[finding_id] = call.kwargs["push_to_jira"]
+        findings_by_title = {finding.title: finding for finding in new_findings}
+        return flag_by_finding_id, findings_by_title
+
+    def test_batch_push_to_jira_last_finding_grouped(self):
+        # Last finding of the batch is grouped -> its False flag must NOT leak onto the
+        # ungrouped finding processed before it
+        ungrouped = Finding(title="Ungrouped Finding", severity="Medium", description="no component")
+        grouped = Finding(title="Grouped Finding", severity="Medium", description="has component", component_name="lib-a")
+        flags, findings_by_title = self._process_findings_with_groups([ungrouped, grouped])
+        grouped_db = findings_by_title["Grouped Finding"]
+        ungrouped_db = findings_by_title["Ungrouped Finding"]
+        self.assertFalse(
+            flags[grouped_db.id],
+            msg=f"grouped finding must not be pushed individually, dispatched with push_to_jira={flags[grouped_db.id]}",
+        )
+        self.assertTrue(
+            flags[ungrouped_db.id],
+            msg=f"ungrouped finding must be pushed individually, dispatched with push_to_jira={flags[ungrouped_db.id]}",
+        )
+
+    def test_batch_push_to_jira_last_finding_ungrouped(self):
+        # Last finding of the batch is ungrouped -> its True flag must NOT leak onto the
+        # grouped finding processed before it
+        grouped = Finding(title="Grouped Finding 2", severity="Medium", description="has component", component_name="lib-b")
+        ungrouped = Finding(title="Ungrouped Finding 2", severity="Medium", description="no component")
+        flags, findings_by_title = self._process_findings_with_groups([grouped, ungrouped])
+        grouped_db = findings_by_title["Grouped Finding 2"]
+        ungrouped_db = findings_by_title["Ungrouped Finding 2"]
+        self.assertFalse(
+            flags[grouped_db.id],
+            msg=f"grouped finding must not be pushed individually, dispatched with push_to_jira={flags[grouped_db.id]}",
+        )
+        self.assertTrue(
+            flags[ungrouped_db.id],
+            msg=f"ungrouped finding must be pushed individually, dispatched with push_to_jira={flags[ungrouped_db.id]}",
+        )