Merge branch 'bugfix' into sc-13470_defender_cloud_docs

Maffooch · web-flow · commit 5b8a8b1d8ef4 · 2026-07-09T11:36:54.000-06:00
diff --git a/dojo/importers/default_importer.py b/dojo/importers/default_importer.py
@@ -182,8 +182,12 @@ def _process_findings_internal(
         parsed_findings: list[Finding],
         **kwargs: dict,
     ) -> list[Finding]:
-        # Batched post-processing (no chord): dispatch a task per 1000 findings or on final finding
-        batch_finding_ids: list[int] = []
+        # Batched post-processing (no chord): dispatch a task per 1000 findings or on final finding.
+        # Each entry carries the finding's own push_to_jira flag: grouped findings are pushed to
+        # JIRA as a group after the loop, so their individual push is suppressed while ungrouped
+        # findings in the same batch must still be pushed. The batch is partitioned by flag at
+        # dispatch time instead of applying one finding's flag to the whole batch.
+        batch_finding_ids: list[tuple[int, bool]] = []
         batch_findings: list[Finding] = []
         batch_max_size = getattr(settings, "IMPORT_REIMPORT_DEDUPE_BATCH_SIZE", 1000)
 
@@ -274,7 +278,7 @@ def _process_findings_internal(
                          self.push_to_jira, self.findings_groups_enabled, self.group_by)
             push_to_jira = self.push_to_jira and ((not self.findings_groups_enabled or not self.group_by) or not finding_will_be_grouped)
             logger.debug("process_findings: computed push_to_jira=%s", push_to_jira)
-            batch_finding_ids.append(finding.id)
+            batch_finding_ids.append((finding.id, push_to_jira))
             batch_findings.append(finding)
 
             # If batch is full or we're at the end, persist locations/endpoints and dispatch
@@ -293,25 +297,31 @@ def _process_findings_internal(
                 # dispatches, so rules/dedup see inherited tags on .tags.
                 apply_inherited_tags_for_findings(batch_findings)
                 batch_findings.clear()
-                finding_ids_batch = list(batch_finding_ids)
+                # Partition the batch by each finding's own push_to_jira flag so one
+                # finding's grouping state is not applied to the whole batch. Uniform
+                # batches (grouping disabled, or push_to_jira off) stay a single dispatch.
+                finding_ids_by_push: dict[bool, list[int]] = {}
+                for finding_id, finding_push_to_jira in batch_finding_ids:
+                    finding_ids_by_push.setdefault(finding_push_to_jira, []).append(finding_id)
                 batch_finding_ids.clear()
-                logger.debug("process_findings: dispatching batch with push_to_jira=%s (batch_size=%d, is_final=%s)",
-                             push_to_jira, len(finding_ids_batch), is_final_finding)
-                result = dojo_dispatch_task(
-                    finding_helper.post_process_findings_batch,
-                    finding_ids_batch,
-                    dedupe_option=True,
-                    rules_option=True,
-                    product_grading_option=True,
-                    issue_updater_option=True,
-                    push_to_jira=push_to_jira,
-                    # 'async_wait' joins on this dispatch via AsyncResult.get(), so its
-                    # result must be stored despite the global CELERY_TASK_IGNORE_RESULT.
-                    **({"ignore_result": False} if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT else {}),
-                    **self.post_processing_dispatch_kwargs(**kwargs),
-                )
-                if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT:
-                    self.record_post_processing_result(result)
+                for push_to_jira_batch, finding_ids_batch in finding_ids_by_push.items():
+                    logger.debug("process_findings: dispatching batch with push_to_jira=%s (batch_size=%d, is_final=%s)",
+                                 push_to_jira_batch, len(finding_ids_batch), is_final_finding)
+                    result = dojo_dispatch_task(
+                        finding_helper.post_process_findings_batch,
+                        finding_ids_batch,
+                        dedupe_option=True,
+                        rules_option=True,
+                        product_grading_option=True,
+                        issue_updater_option=True,
+                        push_to_jira=push_to_jira_batch,
+                        # 'async_wait' joins on this dispatch via AsyncResult.get(), so its
+                        # result must be stored despite the global CELERY_TASK_IGNORE_RESULT.
+                        **({"ignore_result": False} if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT else {}),
+                        **self.post_processing_dispatch_kwargs(**kwargs),
+                    )
+                    if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT:
+                        self.record_post_processing_result(result)
 
             # No chord: tasks are dispatched immediately above per batch
 
@@ -402,7 +412,7 @@ def close_old_findings(
         # Remove all the findings that are coming from the report already mitigated
         new_hash_codes = []
         new_unique_ids_from_tool = []
-        for finding in findings.values():
+        for finding in findings.values("is_mitigated", "hash_code", "unique_id_from_tool"):
             # Do not process closed findings in the report
             if finding.get("is_mitigated", False):
                 continue
diff --git a/dojo/importers/default_reimporter.py b/dojo/importers/default_reimporter.py
@@ -320,7 +320,11 @@ def _process_findings_internal(
                 continue
             cleaned_findings.append(sanitized)
 
-        batch_finding_ids: list[int] = []
+        # Each entry carries the finding's own push_to_jira flag: grouped findings are pushed to
+        # JIRA as a group, so their individual push is suppressed while ungrouped findings in the
+        # same batch must still be pushed. The batch is partitioned by flag at dispatch time
+        # instead of applying one finding's flag to the whole batch.
+        batch_finding_ids: list[tuple[int, bool]] = []
         batch_findings: list[Finding] = []
         # Findings that were newly created (else branch below) — pass these to
         # `apply_inherited_tags_for_findings` instead of `batch_findings` so
@@ -423,7 +427,7 @@ def _process_findings_internal(
                     )
                     # all data is already saved on the finding, we only need to trigger post processing in batches
                     push_to_jira = self.push_to_jira and ((not self.findings_groups_enabled or not self.group_by) or not finding_will_be_grouped)
-                    batch_finding_ids.append(finding.id)
+                    batch_finding_ids.append((finding.id, push_to_jira))
                     batch_findings.append(finding)
 
                     # Post-processing batches (deduplication, rules, etc.) are separate from matching batches.
@@ -460,24 +464,30 @@ def _process_findings_internal(
                         apply_inherited_tags_for_findings(new_findings_in_batch)
                         new_findings_in_batch.clear()
                         batch_findings.clear()
-                        finding_ids_batch = list(batch_finding_ids)
+                        # Partition the batch by each finding's own push_to_jira flag so one
+                        # finding's grouping state is not applied to the whole batch. Uniform
+                        # batches (grouping disabled, or push_to_jira off) stay a single dispatch.
+                        finding_ids_by_push: dict[bool, list[int]] = {}
+                        for finding_id, finding_push_to_jira in batch_finding_ids:
+                            finding_ids_by_push.setdefault(finding_push_to_jira, []).append(finding_id)
                         batch_finding_ids.clear()
-                        result = dojo_dispatch_task(
-                            finding_helper.post_process_findings_batch,
-                            finding_ids_batch,
-                            dedupe_option=True,
-                            rules_option=True,
-                            product_grading_option=True,
-                            issue_updater_option=True,
-                            push_to_jira=push_to_jira,
-                            jira_instance_id=getattr(self.jira_instance, "id", None),
-                            # 'async_wait' joins on this dispatch via AsyncResult.get(), so its
-                            # result must be stored despite the global CELERY_TASK_IGNORE_RESULT.
-                            **({"ignore_result": False} if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT else {}),
-                            **self.post_processing_dispatch_kwargs(**kwargs),
-                        )
-                        if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT:
-                            self.record_post_processing_result(result)
+                        for push_to_jira_batch, finding_ids_batch in finding_ids_by_push.items():
+                            result = dojo_dispatch_task(
+                                finding_helper.post_process_findings_batch,
+                                finding_ids_batch,
+                                dedupe_option=True,
+                                rules_option=True,
+                                product_grading_option=True,
+                                issue_updater_option=True,
+                                push_to_jira=push_to_jira_batch,
+                                jira_instance_id=getattr(self.jira_instance, "id", None),
+                                # 'async_wait' joins on this dispatch via AsyncResult.get(), so its
+                                # result must be stored despite the global CELERY_TASK_IGNORE_RESULT.
+                                **({"ignore_result": False} if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT else {}),
+                                **self.post_processing_dispatch_kwargs(**kwargs),
+                            )
+                            if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT:
+                                self.record_post_processing_result(result)
 
         # No chord: tasks are dispatched immediately above per batch
 
diff --git a/dojo/middleware.py b/dojo/middleware.py
@@ -312,6 +312,15 @@ def install_intermediate_flush_hook():
 
     def add_to_context_with_flush(self, engine, obj):
         original_add(self, engine, obj)
+        # The intermediate flush is a request-path optimization on the global singleton
+        # context (AsyncSearchContextMiddleware). The async reindex task
+        # update_watson_search_index_for_model() builds its OWN SearchContextManager and
+        # IS the drain target -- if it re-drained its batch it would dispatch a clone of
+        # itself, discard those pks unindexed, and loop forever (queue ~0, worker pegged,
+        # nothing indexed). Only the singleton intermediate-flushes; any ad-hoc context
+        # manager indexes its own batch on end().
+        if self is not search_context_manager:
+            return
         threshold = getattr(settings, "WATSON_ASYNC_INDEX_UPDATE_BATCH_SIZE", 1000)
         if threshold <= 0 or not self._stack:
             return
diff --git a/unittests/test_importers_closeold.py b/unittests/test_importers_closeold.py
@@ -1,4 +1,5 @@
 import logging
+from unittest import mock
 
 from django.utils import timezone
 
@@ -12,6 +13,61 @@
 
 
 class TestDojoCloseOld(DojoTestCase):
+    # Regression: close_old_findings fetched every Finding column (incl. description) via
+    # findings.values() just to collect is_mitigated/hash_code/unique_id_from_tool.
+    def test_close_old_findings_only_fetches_needed_columns(self):
+        scan_type = "Acunetix Scan"
+        user, _ = User.objects.get_or_create(username="admin")
+        product_type, _ = Product_Type.objects.get_or_create(name="closeold")
+        environment, _ = Development_Environment.objects.get_or_create(name="Development")
+        product, _ = Product.objects.get_or_create(
+            name="TestDojoCloseOldColumns",
+            description="Test",
+            prod_type=product_type,
+        )
+        engagement, _ = Engagement.objects.get_or_create(
+            name="Close Old Needed Columns",
+            product=product,
+            target_start=timezone.now(),
+            target_end=timezone.now(),
+        )
+        import_options = {
+            "user": user,
+            "lead": user,
+            "scan_date": None,
+            "environment": environment,
+            "active": True,
+            "verified": False,
+            "engagement": engagement,
+            "scan_type": scan_type,
+        }
+        # Seed an older test with 4 findings that the close-old pass should mitigate
+        with (get_unit_tests_scans_path("acunetix") / "many_findings.xml").open(encoding="utf-8") as many_findings_scan:
+            importer = DefaultImporter(close_old_findings=False, **import_options)
+            _, _, len_new_findings, _, _, _, _ = importer.process_scan(many_findings_scan)
+            self.assertEqual(4, len_new_findings)
+        # Import a report with a single finding, without closing, to get the new test
+        with (get_unit_tests_scans_path("acunetix") / "one_finding.xml").open(encoding="utf-8") as single_finding_scan:
+            importer = DefaultImporter(close_old_findings=False, **import_options)
+            test, _, len_new_findings, _, _, _, _ = importer.process_scan(single_finding_scan)
+            self.assertEqual(1, len_new_findings)
+        # Call close_old_findings directly with the new test's queryset and spy on values()
+        importer = DefaultImporter(close_old_findings=True, **import_options)
+        importer.test = test
+        findings_queryset = test.finding_set.all()
+        with mock.patch.object(findings_queryset, "values", wraps=findings_queryset.values) as values_spy:
+            closed_findings = importer.close_old_findings(findings_queryset)
+        values_spy.assert_called_once_with(
+            "is_mitigated", "hash_code", "unique_id_from_tool",
+        )
+        # Control: narrowing the columns must not change which findings get closed
+        # (the 4 findings from the older test; dedupe is off, so the overlapping
+        # finding closes too — same semantics as test_close_old_same_engagement)
+        self.assertEqual(
+            4, len(closed_findings),
+            msg=f"expected 4 old findings closed, got {len(closed_findings)}",
+        )
+
     def test_close_old_same_engagement(self):
         scan_type = "Acunetix Scan"
         user, _ = User.objects.get_or_create(username="admin")
diff --git a/unittests/test_importers_importer.py b/unittests/test_importers_importer.py
@@ -1143,3 +1143,91 @@ def test_reactivation_of_non_duplicate_still_activates(self):
         self.assertIsNone(result_finding.mitigated)
         self.assertTrue(result_finding.active)
         self.assertTrue(result_finding.verified)
+
+
+# Regression: the per-batch post-processing dispatch read the push_to_jira flag computed
+# for the LAST finding of the batch and applied it to the entire batch. With finding
+# groups enabled and push_to_jira=True, a mixed batch ending in a grouped finding
+# suppressed the JIRA push for every ungrouped finding in the batch (and vice versa).
+class TestDojoImporterBatchPushToJira(DojoTestCase):
+    def _process_findings_with_groups(self, parsed_findings):
+        """Run process_findings with push_to_jira + group_by and capture per-finding dispatch flags."""
+        scan_type = "Acunetix Scan"
+        user, _ = User.objects.get_or_create(username="admin")
+        product_type, _ = Product_Type.objects.get_or_create(name="batch_push")
+        product, _ = Product.objects.get_or_create(
+            name="TestDojoImporterBatchPushToJira",
+            description="test product",
+            prod_type=product_type,
+        )
+        engagement, _ = Engagement.objects.get_or_create(
+            name="Batch Push To Jira",
+            product=product,
+            target_start=timezone.now(),
+            target_end=timezone.now(),
+        )
+        environment, _ = Development_Environment.objects.get_or_create(name="Development")
+        self.system_settings(enable_finding_groups=True)
+        importer = DefaultImporter(
+            user=user,
+            lead=user,
+            scan_date=None,
+            environment=environment,
+            minimum_severity="Info",
+            active=True,
+            verified=True,
+            scan_type=scan_type,
+            engagement=engagement,
+            push_to_jira=True,
+            group_by="component_name",
+        )
+        test = importer.create_test(scan_type)
+        for finding in parsed_findings:
+            finding.test = test
+        with (
+            patch("dojo.importers.default_importer.dojo_dispatch_task") as dispatch_mock,
+            patch("dojo.importers.default_importer.jira_services.push"),
+        ):
+            new_findings = importer.process_findings(parsed_findings)
+        # Map each dispatched finding id to the push_to_jira flag its batch was sent with
+        flag_by_finding_id = {}
+        for call in dispatch_mock.call_args_list:
+            finding_ids = call.args[1]
+            for finding_id in finding_ids:
+                flag_by_finding_id[finding_id] = call.kwargs["push_to_jira"]
+        findings_by_title = {finding.title: finding for finding in new_findings}
+        return flag_by_finding_id, findings_by_title
+
+    def test_batch_push_to_jira_last_finding_grouped(self):
+        # Last finding of the batch is grouped -> its False flag must NOT leak onto the
+        # ungrouped finding processed before it
+        ungrouped = Finding(title="Ungrouped Finding", severity="Medium", description="no component")
+        grouped = Finding(title="Grouped Finding", severity="Medium", description="has component", component_name="lib-a")
+        flags, findings_by_title = self._process_findings_with_groups([ungrouped, grouped])
+        grouped_db = findings_by_title["Grouped Finding"]
+        ungrouped_db = findings_by_title["Ungrouped Finding"]
+        self.assertFalse(
+            flags[grouped_db.id],
+            msg=f"grouped finding must not be pushed individually, dispatched with push_to_jira={flags[grouped_db.id]}",
+        )
+        self.assertTrue(
+            flags[ungrouped_db.id],
+            msg=f"ungrouped finding must be pushed individually, dispatched with push_to_jira={flags[ungrouped_db.id]}",
+        )
+
+    def test_batch_push_to_jira_last_finding_ungrouped(self):
+        # Last finding of the batch is ungrouped -> its True flag must NOT leak onto the
+        # grouped finding processed before it
+        grouped = Finding(title="Grouped Finding 2", severity="Medium", description="has component", component_name="lib-b")
+        ungrouped = Finding(title="Ungrouped Finding 2", severity="Medium", description="no component")
+        flags, findings_by_title = self._process_findings_with_groups([grouped, ungrouped])
+        grouped_db = findings_by_title["Grouped Finding 2"]
+        ungrouped_db = findings_by_title["Ungrouped Finding 2"]
+        self.assertFalse(
+            flags[grouped_db.id],
+            msg=f"grouped finding must not be pushed individually, dispatched with push_to_jira={flags[grouped_db.id]}",
+        )
+        self.assertTrue(
+            flags[ungrouped_db.id],
+            msg=f"ungrouped finding must be pushed individually, dispatched with push_to_jira={flags[ungrouped_db.id]}",
+        )
diff --git a/unittests/test_watson_intermediate_flush.py b/unittests/test_watson_intermediate_flush.py
