diff --git a/docs/content/releases/os_upgrading/2.55.2.md b/docs/content/releases/os_upgrading/2.55.2.md new file mode 100644 index 00000000000..45df3e68dc7 --- /dev/null +++ b/docs/content/releases/os_upgrading/2.55.2.md @@ -0,0 +1,55 @@ +--- +title: 'Upgrading to DefectDojo Version 2.55.2' +toc_hide: true +weight: -20260208 +description: JIRA Reconciliation now also processes Finding Groups. +--- + +## JIRA Reconciliation + +The `jira_status_reconciliation` management command now also processes JIRA issues for Finding Groups. + +New command line options: + +- `--include-findings` / `--no-include-findings` — Process individual findings with direct JIRA issues (default: True) +- `--include-finding-groups` / `--no-include-finding-groups` — Process finding groups with JIRA issues (default: True) + +Full list of options: + + docker compose exec uwsgi bash -c "python manage.py jira_status_reconciliation --help" + + usage: manage.py jira_status_reconciliation [-h] [--mode MODE] [--product PRODUCT] + [--engagement ENGAGEMENT] [--daysback DAYSBACK] [--dryrun] + [--include-findings | --no-include-findings] + [--include-finding-groups | --no-include-finding-groups] + [--version] [-v {0,1,2,3}] [--settings SETTINGS] + [--pythonpath PYTHONPATH] [--traceback] [--no-color] + [--force-color] [--skip-checks] + + Reconcile finding/finding group status with JIRA issue status, stdout will + contain semicolon separated CSV results. Risk Accepted findings are skipped. + Findings created before 1.14.0 are skipped. + + options: + -h, --help show this help message and exit + --mode MODE reconcile: (default) reconcile any differences in + status between Defect Dojo and JIRA. + push_status_to_jira: update JIRA status for all JIRA + issues connected to a finding or finding group. + import_status_from_jira: update finding/finding group + status from JIRA. + --product PRODUCT Only process findings in this product (name) + --engagement ENGAGEMENT + Only process findings in this engagement (name) + --daysback DAYSBACK Only process findings created in the last + 'daysback' days + --dryrun Only print actions to be performed, but make no + modifications. + --include-findings, --no-include-findings + Process individual findings with direct JIRA issues + (default: True) + --include-finding-groups, --no-include-finding-groups + Process finding groups with JIRA issues + (default: True) + +Check the [Release Notes](https://github.com/DefectDojo/django-DefectDojo/releases/tag/2.55.2) for the contents of the release. diff --git a/dojo/management/commands/jira_status_reconciliation.py b/dojo/management/commands/jira_status_reconciliation.py index a6a49e9256b..2b8a649ed9f 100644 --- a/dojo/management/commands/jira_status_reconciliation.py +++ b/dojo/management/commands/jira_status_reconciliation.py @@ -1,3 +1,4 @@ +import argparse import logging import pghistory @@ -8,7 +9,7 @@ from django.utils.dateparse import parse_datetime import dojo.jira_link.helper as jira_helper -from dojo.models import Engagement, Finding, Product +from dojo.models import Engagement, Finding, Finding_Group, Product logger = logging.getLogger(__name__) @@ -19,6 +20,8 @@ def jira_status_reconciliation(*args, **kwargs): engagement = kwargs["engagement"] daysback = kwargs["daysback"] dryrun = kwargs["dryrun"] + include_findings = kwargs.get("include_findings", True) + include_finding_groups = kwargs.get("include_finding_groups", True) logger.debug("mode: %s product:%s engagement: %s dryrun: %s", mode, product, engagement, dryrun) @@ -29,22 +32,50 @@ def jira_status_reconciliation(*args, **kwargs): if not mode: mode = "reconcile" - findings = Finding.objects.all() + # Resolve product and engagement objects once for reuse in both loops + product_obj = None if product: - product = Product.objects.filter(name=product).first() - findings = findings.filter(test__engagement__product=product) + product_obj = Product.objects.filter(name=product).first() + engagement_obj = None if engagement: - engagement = Engagement.objects.filter(name=engagement).first() - findings = findings.filter(test__engagement=engagement) + engagement_obj = Engagement.objects.filter(name=engagement).first() + timestamp = None if daysback: timestamp = timezone.now() - relativedelta(days=int(daysback)) + + messages = ["jira_key;url;resolution_or_status;jira_issue.jira_change;issue_from_jira.fields.updated;last_status_update;issue_from_jira.fields.updated;last_reviewed;issue_from_jira.fields.updated;flag1;flag2;flag3;action;change_made"] + + # --- Process individual findings with direct JIRA issues --- + if include_findings: + _reconcile_findings(mode, product_obj, engagement_obj, timestamp, dryrun, messages) + + # --- Process finding groups with JIRA issues --- + if include_finding_groups: + _reconcile_finding_groups(mode, product_obj, engagement_obj, timestamp, dryrun, messages) + + logger.info("results (semicolon seperated)") + for message in messages: + logger.info(message) + return None + + +def _reconcile_findings(mode, product_obj, engagement_obj, timestamp, dryrun, messages): + """Reconcile individual findings that have their own direct JIRA issues.""" + findings = Finding.objects.all() + if product_obj: + findings = findings.filter(test__engagement__product=product_obj) + + if engagement_obj: + findings = findings.filter(test__engagement=engagement_obj) + + if timestamp: findings = findings.filter(created__gte=timestamp) findings = findings.exclude(jira_issue__isnull=True) - # order by product, engagement to increase the cance of being able to reuse jira_instance + jira connection + # order by product, engagement to increase the chance of being able to reuse jira_instance + jira connection findings = findings.order_by("test__engagement__product__id", "test__engagement__id") findings = findings.prefetch_related("jira_issue__jira_project__jira_instance") @@ -53,7 +84,6 @@ def jira_status_reconciliation(*args, **kwargs): logger.debug(findings.query) - messages = ["jira_key;finding_url;resolution_or_status;find.jira_issue.jira_change;issue_from_jira.fields.updated;find.last_status_update;issue_from_jira.fields.updated;find.last_reviewed;issue_from_jira.fields.updated;flag1;flag2;flag3;action;change_made"] for find in findings: logger.debug("jira status reconciliation for: %i:%s", find.id, find) @@ -182,10 +212,171 @@ def jira_status_reconciliation(*args, **kwargs): logger.info(message) - logger.info("results (semicolon seperated)") - for message in messages: - logger.info(message) - return None + +def _reconcile_finding_groups(mode, product_obj, engagement_obj, timestamp, dryrun, messages): + """ + Reconcile finding groups that have their own JIRA issues. + + This handles JIRA issues attached to Finding_Group objects separately from + individual finding JIRA issues to avoid pushing the same JIRA issue twice. + We use push_status_to_jira directly on the group (not push_finding_group_to_jira + which would also push individual finding JIRA issues already handled by + _reconcile_findings). + """ + finding_groups = Finding_Group.objects.all() + if product_obj: + finding_groups = finding_groups.filter(test__engagement__product=product_obj) + + if engagement_obj: + finding_groups = finding_groups.filter(test__engagement=engagement_obj) + + if timestamp: + finding_groups = finding_groups.filter(created__gte=timestamp) + + finding_groups = finding_groups.exclude(jira_issue__isnull=True) + + # order by product, engagement to increase the chance of being able to reuse jira_instance + jira connection + finding_groups = finding_groups.order_by("test__engagement__product__id", "test__engagement__id") + + finding_groups = finding_groups.prefetch_related("jira_issue__jira_project__jira_instance") + finding_groups = finding_groups.prefetch_related("test__engagement__jira_project__jira_instance") + finding_groups = finding_groups.prefetch_related("test__engagement__product__jira_project_set__jira_instance") + finding_groups = finding_groups.prefetch_related("findings") + + logger.debug(finding_groups.query) + + for finding_group in finding_groups: + logger.debug("jira status reconciliation for finding group: %i:%s", finding_group.id, finding_group) + + group_findings = finding_group.findings.all() + group_url = f"{settings.SITE_URL}/test/{finding_group.test.id}" + + issue_from_jira = jira_helper.get_jira_issue_from_jira(finding_group) + + if not issue_from_jira: + message = f"{finding_group.jira_issue.jira_key};{group_url};{finding_group.status()};unable to retrieve JIRA Issue;error" + messages.append(message) + logger.info(message) + continue + + assignee = issue_from_jira.fields.assignee if hasattr(issue_from_jira.fields, "assignee") else None + assignee_name = assignee.displayName if assignee else None + resolution = issue_from_jira.fields.resolution if issue_from_jira.fields.resolution and issue_from_jira.fields.resolution != "None" else None + resolution_id = resolution.id if resolution else None + resolution_name = resolution.name if resolution else None + + # convert from str to datetime + issue_from_jira.fields.updated = parse_datetime(issue_from_jira.fields.updated) + + # Derive timestamps from the findings in the group + group_last_status_update = _max_or_none(f.last_status_update for f in group_findings) + group_last_reviewed = _max_or_none(f.last_reviewed for f in group_findings) + group_is_active = any(f.active for f in group_findings) + group_all_mitigated = all(f.is_mitigated for f in group_findings) if group_findings else False + + flag1, flag2, flag3 = None, None, None + + if mode == "reconcile" and not group_last_status_update: + message = f"{finding_group.jira_issue.jira_key}; {group_url};finding_group:{finding_group.id};{finding_group.status()};skipping finding group with no last_status_update;skipped" + messages.append(message) + logger.info(message) + continue + + jira_is_active = jira_helper.issue_from_jira_is_active(issue_from_jira) + + if jira_is_active and group_is_active: + message = f"{finding_group.jira_issue.jira_key}; {group_url};finding_group:{finding_group.id};{finding_group.status()};{resolution_name};no action both sides are active/open;equal" + messages.append(message) + logger.info(message) + elif not jira_is_active and not group_is_active: + message = f"{finding_group.jira_issue.jira_key}; {group_url};finding_group:{finding_group.id};{finding_group.status()};{resolution_name};no action both sides are inactive/closed;equal" + messages.append(message) + logger.info(message) + + else: + # statuses are different + action = None + if mode in {"push_status_to_jira", "import_status_from_jira"}: + action = mode + else: + # reconcile - determine which side is newer using derived timestamps + # Status in JIRA is newer if all DefectDojo timestamps are older than JIRA updated + flag1 = (not finding_group.jira_issue.jira_change or (finding_group.jira_issue.jira_change < issue_from_jira.fields.updated)) + flag2 = not group_last_status_update or (group_last_status_update < issue_from_jira.fields.updated) + flag3 = (not group_last_reviewed or (group_last_reviewed < issue_from_jira.fields.updated)) + + logger.debug("finding_group reconcile: %s,%s,%s,%s", resolution_name, flag1, flag2, flag3) + + if flag1 and flag2 and flag3: + action = "import_status_from_jira" + else: + # Status in DefectDojo is newer + flag1 = not finding_group.jira_issue.jira_change or (finding_group.jira_issue.jira_change > issue_from_jira.fields.updated) + flag2 = group_last_status_update and (group_last_status_update > issue_from_jira.fields.updated) + flag3 = group_all_mitigated and finding_group.jira_issue.jira_change and any( + f.is_mitigated and f.mitigated and f.mitigated > finding_group.jira_issue.jira_change + for f in group_findings + ) + + logger.debug("finding_group reconcile dojo newer: %s,%s,%s,%s", resolution_name, flag1, flag2, flag3) + + if flag1 or flag2 or flag3: + action = "push_status_to_jira" + + prev_jira_instance, jira = None, None + + if action == "import_status_from_jira": + # Import status from JIRA to all findings in the group + # Same pattern as the JIRA webhook handler in dojo/jira_link/views.py + any_status_changed = False + for find in group_findings: + if not dryrun: + status_changed = jira_helper.process_resolution_from_jira( + find, resolution_id, resolution_name, assignee_name, + issue_from_jira.fields.updated, finding_group.jira_issue, + finding_group=finding_group, + ) + else: + status_changed = "dryrun" + if status_changed: + any_status_changed = True + + message_action = "deactivating" if group_is_active else "reactivating" + message = f"{finding_group.jira_issue.jira_key}; {group_url};finding_group:{finding_group.id};{finding_group.status()};{resolution_name};{flag1};{flag2};{flag3};{message_action} findings in finding group;{any_status_changed}" + messages.append(message) + logger.info(message) + + elif action == "push_status_to_jira": + # Push the finding group's aggregate status to its JIRA issue directly. + # We do NOT use push_finding_group_to_jira here because that would also push + # individual finding JIRA issues which are already handled by _reconcile_findings. + jira_instance = jira_helper.get_jira_instance(finding_group) + if not prev_jira_instance or (jira_instance.id != prev_jira_instance.id): + jira = jira_helper.get_jira_connection(jira_instance) + + message_action = "reopening" if group_is_active else "closing" + + status_changed = jira_helper.push_status_to_jira(finding_group, jira_instance, jira, issue_from_jira, save=True) if not dryrun else "dryrun" + + if status_changed: + message = f"{finding_group.jira_issue.jira_key}; {group_url};finding_group:{finding_group.id};{finding_group.status()};{resolution_name};{flag1};{flag2};{flag3};{message_action} jira issue for finding group;{status_changed}" + else: + if status_changed is None: + status_changed = "Error" + message = f"{finding_group.jira_issue.jira_key}; {group_url};finding_group:{finding_group.id};{finding_group.status()};{resolution_name};{flag1};{flag2};{flag3};no changes made while pushing status to jira;{status_changed}" + + messages.append(message) + logger.info(message) + else: + message = f"{finding_group.jira_issue.jira_key}; {group_url};finding_group:{finding_group.id};{finding_group.status()};{resolution_name};{flag1};{flag2};{flag3};unable to determine source of truth;unknown" + messages.append(message) + logger.info(message) + + +def _max_or_none(iterable): + """Return the max of non-None values in iterable, or None if all are None.""" + values = [v for v in iterable if v is not None] + return max(values) if values else None class Command(BaseCommand): @@ -200,21 +391,29 @@ class Command(BaseCommand): - sync_from_jira: overwrite status in Defect Dojo with status from JIRA """ - help = "Reconcile finding status with JIRA issue status, stdout will contain semicolon seperated CSV results. \ + help = "Reconcile finding/finding group status with JIRA issue status, stdout will contain semicolon seperated CSV results. \ Risk Accepted findings are skipped. Findings created before 1.14.0 are skipped." mode_help = ( "- reconcile: (default)reconcile any differences in status between Defect Dojo and JIRA, will look at the latest status change timestamp in both systems to determine which one is the correct status" - "- push_status_to_jira: update JIRA status for all JIRA issues connected to a Defect Dojo finding (will not push summary/description, only status)" - "- import_status_from_jira: update Defect Dojo finding status from JIRA" + "- push_status_to_jira: update JIRA status for all JIRA issues connected to a Defect Dojo finding or finding group (will not push summary/description, only status)" + "- import_status_from_jira: update Defect Dojo finding/finding group status from JIRA" ) def add_arguments(self, parser): parser.add_argument("--mode", help=self.mode_help) parser.add_argument("--product", help="Only process findings in this product (name)") - parser.add_argument("--engagement", help="Only process findings in this product (name)") + parser.add_argument("--engagement", help="Only process findings in this engagement (name)") parser.add_argument("--daysback", type=int, help="Only process findings created in the last 'daysback' days") parser.add_argument("--dryrun", action="store_true", help="Only print actions to be performed, but make no modifications.") + parser.add_argument( + "--include-findings", action=argparse.BooleanOptionalAction, default=True, + help="Process individual findings with direct JIRA issues (default: True)", + ) + parser.add_argument( + "--include-finding-groups", action=argparse.BooleanOptionalAction, default=True, + help="Process finding groups with JIRA issues (default: True)", + ) def handle(self, *args, **options): # Wrap with pghistory context for audit trail