push_to_jira: fix pushing to JIRA during import/reimport in asynchronous mode (DefectDojo#13916)

* push_to_jira: add logging

* push_to_jira: add logging

* push to jira: fix passing of parameters in async mode

* push to jira: fix passing of parameters in async mode

Author: valentijnscholten

dojo/api_v2/serializers.py

@@ -2281,6 +2281,7 @@ def process_scan(
         Raises exceptions in the event of an error
         """
         try:
+            logger.debug(f"process_scan called with context: {context}")
             start_time = time.perf_counter()
             importer = self.get_importer(**context)
             context["test"], _, _, _, _, _, _ = importer.process_scan(
@@ -2558,6 +2559,7 @@ def process_scan(
         """
         statistics_before, statistics_delta = None, None
         try:
+            logger.debug(f"process_scan called with context: {context}")
             start_time = time.perf_counter()
             if test := context.get("test"):
                 statistics_before = test.statistics

dojo/api_v2/views.py

@@ -2515,7 +2515,7 @@ def perform_create(self, serializer):
             jira_driver = engagement or (product or None)
             if jira_project := (jira_helper.get_jira_project(jira_driver) if jira_driver else None):
                 push_to_jira = push_to_jira or jira_project.push_all_issues
-        # logger.debug(f"push_to_jira: {push_to_jira}")
+
         serializer.save(push_to_jira=push_to_jira)
 
     def get_queryset(self):

dojo/engagement/views.py

@@ -936,6 +936,30 @@ def import_findings(
     ) -> str | None:
         """Attempt to import with all the supplied information"""
         try:
+            # Log only user-entered form values, excluding internal objects
+            user_values = {
+                "scan_type": context.get("scan_type"),
+                "scan_date": context.get("scan_date"),
+                "minimum_severity": context.get("minimum_severity"),
+                "active": context.get("active"),
+                "verified": context.get("verified"),
+                "test_title": context.get("test_title"),
+                "tags": context.get("tags"),
+                "version": context.get("version"),
+                "branch_tag": context.get("branch_tag"),
+                "build_id": context.get("build_id"),
+                "commit_hash": context.get("commit_hash"),
+                "service": context.get("service"),
+                "close_old_findings": context.get("close_old_findings"),
+                "apply_tags_to_findings": context.get("apply_tags_to_findings"),
+                "apply_tags_to_endpoints": context.get("apply_tags_to_endpoints"),
+                "close_old_findings_product_scope": context.get("close_old_findings_product_scope"),
+                "group_by": context.get("group_by"),
+                "create_finding_groups_for_all_findings": context.get("create_finding_groups_for_all_findings"),
+                "push_to_jira": context.get("push_to_jira"),
+                "push_all_jira_issues": context.get("push_all_jira_issues"),
+            }
+            logger.debug(f"import_findings called with user values: {user_values}")
             importer_client = self.get_importer(context)
             context["test"], _, finding_count, closed_finding_count, _, _, _ = importer_client.process_scan(
                 context.pop("scan", None),

dojo/finding/deduplication.py

@@ -27,6 +27,7 @@ def get_finding_models_for_deduplication(finding_ids):
 
     """
     if not finding_ids:
+        logger.debug("get_finding_models_for_deduplication called with no finding_ids")
         return []
 
     return list(
@@ -543,6 +544,7 @@ def dedupe_batch_of_findings(findings, *args, **kwargs):
         return batch_dedupe_method(findings, *args, **kwargs)
 
     if not findings:
+        logger.debug("dedupe_batch_of_findings called with no findings")
         return None
 
     enabled = System_Settings.objects.get().enable_deduplication

dojo/finding/helper.py

@@ -470,22 +470,30 @@ def post_process_finding_save_internal(finding, dedupe_option=True, rules_option
 @app.task
 def post_process_findings_batch_signature(finding_ids, *args, dedupe_option=True, rules_option=True, product_grading_option=True,
              issue_updater_option=True, push_to_jira=False, user=None, **kwargs):
-    return post_process_findings_batch(finding_ids, dedupe_option, rules_option, product_grading_option,
-                                       issue_updater_option, push_to_jira, user, **kwargs)
+    return post_process_findings_batch(finding_ids, *args, dedupe_option=dedupe_option, rules_option=rules_option, product_grading_option=product_grading_option, issue_updater_option=issue_updater_option, push_to_jira=push_to_jira, user=user, **kwargs)
+    # Pass arguments as keyword arguments to ensure Celery properly serializes them
 
 
 @dojo_async_task
 @app.task
 def post_process_findings_batch(finding_ids, *args, dedupe_option=True, rules_option=True, product_grading_option=True,
              issue_updater_option=True, push_to_jira=False, user=None, **kwargs):
 
+    logger.debug(
+        f"post_process_findings_batch called: finding_ids_count={len(finding_ids) if finding_ids else 0}, "
+        f"args={args}, dedupe_option={dedupe_option}, rules_option={rules_option}, "
+        f"product_grading_option={product_grading_option}, issue_updater_option={issue_updater_option}, "
+        f"push_to_jira={push_to_jira}, user={user.id if user else None}, kwargs={kwargs}",
+    )
     if not finding_ids:
         return
 
     system_settings = System_Settings.objects.get()
 
     # use list() to force a complete query execution and related objects to be loaded once
+    logger.debug(f"getting finding models for batch deduplication with: {len(finding_ids)} findings")
     findings = get_finding_models_for_deduplication(finding_ids)
+    logger.debug(f"found {len(findings)} findings for batch deduplication")
 
     if not findings:
         logger.debug(f"no findings found for batch deduplication with IDs: {finding_ids}")
@@ -517,6 +525,8 @@ def post_process_findings_batch(finding_ids, *args, dedupe_option=True, rules_op
                 jira_helper.push_to_jira(finding)
             else:
                 jira_helper.push_to_jira(finding.finding_group)
+    else:
+        logger.debug("push_to_jira is False, not ushing to JIRA")
 
 
 @receiver(pre_delete, sender=Finding)

dojo/importers/default_importer.py

@@ -238,22 +238,30 @@ def process_findings(
             # Categorize this finding as a new one
             new_findings.append(finding)
             # all data is already saved on the finding, we only need to trigger post processing in batches
+            logger.debug("process_findings: self.push_to_jira=%s, self.findings_groups_enabled=%s, self.group_by=%s",
+                         self.push_to_jira, self.findings_groups_enabled, self.group_by)
             push_to_jira = self.push_to_jira and (not self.findings_groups_enabled or not self.group_by)
+            logger.debug("process_findings: computed push_to_jira=%s", push_to_jira)
             batch_finding_ids.append(finding.id)
 
             # If batch is full or we're at the end, dispatch one batched task
             if len(batch_finding_ids) >= batch_max_size or is_final_finding:
                 finding_ids_batch = list(batch_finding_ids)
                 batch_finding_ids.clear()
+                logger.debug("process_findings: dispatching batch with push_to_jira=%s (batch_size=%d, is_final=%s)",
+                             push_to_jira, len(finding_ids_batch), is_final_finding)
                 if we_want_async(async_user=self.user):
-                    finding_helper.post_process_findings_batch_signature(
+                    signature = finding_helper.post_process_findings_batch_signature(
                         finding_ids_batch,
                         dedupe_option=True,
                         rules_option=True,
                         product_grading_option=True,
                         issue_updater_option=True,
                         push_to_jira=push_to_jira,
-                    )()
+                    )
+                    logger.debug("process_findings: signature created with push_to_jira=%s, signature.kwargs=%s",
+                                 push_to_jira, signature.kwargs)
+                    signature()
                 else:
                     finding_helper.post_process_findings_batch(
                         finding_ids_batch,
@@ -279,6 +287,8 @@ def process_findings(
                     jira_helper.push_to_jira(findings[0].finding_group)
                 else:
                     jira_helper.push_to_jira(findings[0])
+            else:
+                logger.debug("push_to_jira is False, not pushing to JIRA")
 
         # Note: All chord batching is now handled within the loop above
 

dojo/test/views.py

@@ -964,6 +964,30 @@ def reimport_findings(
     ) -> str | None:
         """Attempt to import with all the supplied information"""
         try:
+            # Log only user-entered form values, excluding internal objects
+            user_values = {
+                "scan_type": context.get("scan_type"),
+                "scan_date": context.get("scan_date"),
+                "minimum_severity": context.get("minimum_severity"),
+                "active": context.get("active"),
+                "verified": context.get("verified"),
+                "tags": context.get("tags"),
+                "version": context.get("version"),
+                "branch_tag": context.get("branch_tag"),
+                "build_id": context.get("build_id"),
+                "commit_hash": context.get("commit_hash"),
+                "service": context.get("service"),
+                "close_old_findings": context.get("close_old_findings"),
+                "apply_tags_to_findings": context.get("apply_tags_to_findings"),
+                "apply_tags_to_endpoints": context.get("apply_tags_to_endpoints"),
+                "close_old_findings_product_scope": context.get("close_old_findings_product_scope"),
+                "group_by": context.get("group_by"),
+                "create_finding_groups_for_all_findings": context.get("create_finding_groups_for_all_findings"),
+                "push_to_jira": context.get("push_to_jira"),
+                "push_all_jira_issues": context.get("push_all_jira_issues"),
+                "do_not_reactivate": context.get("do_not_reactivate"),
+            }
+            logger.debug(f"reimport_findings called with user values: {user_values}")
             importer_client = self.get_reimporter(context)
             (
                 context["test"],