add flag gating to checkmarx, risk_recon, acunetix + tests

Author: paulOsinski

dojo/tools/acunetix/parse_acunetix360_json.py

@@ -142,9 +142,8 @@ def get_findings(self, filename, test):
             else:
                 # TODO: Delete this after the move to Locations
                 finding.unsaved_endpoints = [Endpoint.from_uri(url)]
-            if item.get("FirstSeenDate"):
-                parseddate = parser.parse(item["FirstSeenDate"], dayfirst=True)
-                finding.date = parseddate
+            if settings.USE_FIRST_SEEN and item.get("FirstSeenDate"):
+                finding.date = parser.parse(item["FirstSeenDate"], dayfirst=True)
             if dupe_key in dupes:
                 find = dupes[dupe_key]
                 find.unsaved_req_resp.extend(finding.unsaved_req_resp)

dojo/tools/checkmarx/parser.py

@@ -4,6 +4,7 @@
 
 from dateutil import parser
 from defusedxml import ElementTree
+from django.conf import settings
 
 from dojo.models import Finding
 from dojo.utils import add_language
@@ -447,12 +448,14 @@ def _get_findings_json(self, file, test):
                         if query.get("groupName"):
                             query.get("groupName").replace("_", " ")
                         for vulnerability in query.get("vulnerabilities", []):
+                            if settings.USE_FIRST_SEEN:
+                                date = self._parse_date(vulnerability.get("firstFoundDate"))
+                            else:
+                                date = self._parse_date(vulnerability.get("foundDate"))
                             finding = Finding(
                                 description=descriptiondetails,
                                 title=title,
-                                date=self._parse_date(
-                                    vulnerability.get("firstFoundDate"),
-                                ),
+                                date=date,
                                 severity=vulnerability.get("severity").title(),
                                 active=(
                                     vulnerability.get("status")
@@ -486,12 +489,14 @@ def _get_findings_json(self, file, test):
                     component_version = package.get("name").split("-")[-1]
                     for vulnerability in package.get("vulnerabilities", []):
                         cve = vulnerability.get("cveId")
+                        if settings.USE_FIRST_SEEN:
+                            date = self._parse_date(vulnerability.get("firstFoundDate"))
+                        else:
+                            date = self._parse_date(vulnerability.get("foundDate"))
                         finding = Finding(
                             title=f"{component_name}:{component_version} | {cve}",
                             description=vulnerability.get("description"),
-                            date=self._parse_date(
-                                vulnerability.get("firstFoundDate"),
-                            ),
+                            date=date,
                             severity=vulnerability.get("severity").title(),
                             active=(
                                 vulnerability.get("status")
@@ -524,12 +529,14 @@ def _get_findings_json(self, file, test):
                 for kics_type in results[result_type].get("results", []):
                     name = kics_type.get("name")
                     for vulnerability in kics_type.get("vulnerabilities", []):
+                        if settings.USE_FIRST_SEEN:
+                            date = self._parse_date(vulnerability.get("firstFoundDate"))
+                        else:
+                            date = self._parse_date(vulnerability.get("foundDate"))
                         finding = Finding(
                             title=f'{name} | {vulnerability.get("issueType")}',
                             description=vulnerability.get("description"),
-                            date=self._parse_date(
-                                vulnerability.get("firstFoundDate"),
-                            ),
+                            date=date,
                             severity=vulnerability.get("severity").title(),
                             active=(
                                 vulnerability.get("status")

dojo/tools/risk_recon/parser.py

@@ -1,6 +1,7 @@
 import json
 
 import dateutil
+from django.conf import settings
 
 from dojo.models import Finding
 from dojo.tools.risk_recon.api import RiskReconAPI
@@ -80,7 +81,10 @@ def _get_findings_internal(self, findings, test):
             findingdetail += "**Priority:** " + item.get("priority") + "\n"
             findingdetail += "**First Seen:** " + item.get("first_seen") + "\n"
 
-            date = dateutil.parser.parse(item.get("first_seen"))
+            if settings.USE_FIRST_SEEN:
+                date = dateutil.parser.parse(item.get("first_seen"))
+            else:
+                date = dateutil.parser.parse(item.get("last_seen"))
 
             sev = item.get("severity", "").capitalize()
             sev = sev or "Info"

unittests/tools/test_acunetix_parser.py

@@ -1,6 +1,8 @@
 import datetime
 from datetime import datetime as date
 
+from django.test import override_settings
+
 from dojo.models import Test
 from dojo.tools.acunetix.parser import AcunetixParser
 from unittests.dojo_test_case import DojoTestCase, get_unit_tests_scans_path
@@ -198,6 +200,7 @@ def test_parse_file_with_example_com(self):
                 self.assertIsInstance(req_resp["resp"], str)
 
     def test_parse_file_with_one_finding_acunetix360(self):
+        """With USE_FIRST_SEEN=False (default), date should come from Generated (scan date)."""
         with (get_unit_tests_scans_path("acunetix") / "acunetix360_one_finding.json").open(encoding="utf-8") as testfile:
             parser = AcunetixParser()
             findings = parser.get_findings(testfile, Test())
@@ -213,9 +216,22 @@ def test_parse_file_with_one_finding_acunetix360(self):
                 self.assertEqual(1, len(self.get_unsaved_locations(finding)))
                 location = self.get_unsaved_locations(finding)[0]
                 self.assertEqual(str(location), "http://php.testsparker.com/auth/login.php")
-                self.assertEqual(finding.date, date(2021, 6, 16, 12, 30))
+                # Generated date is "25/06/2021 09:59 AM" — with USE_FIRST_SEEN=False, FirstSeenDate is ignored
+                self.assertEqual(finding.date, date(2021, 6, 25, 9, 59))
                 self.assertIn("https://online.acunetix360.com/issues/detail/735f4503-e9eb-4b4c-4306-ad49020a4c4b", finding.references)
 
+    @override_settings(USE_FIRST_SEEN=True)
+    def test_parse_file_with_one_finding_acunetix360_first_seen(self):
+        """With USE_FIRST_SEEN=True, date should come from FirstSeenDate."""
+        with (get_unit_tests_scans_path("acunetix") / "acunetix360_one_finding.json").open(encoding="utf-8") as testfile:
+            parser = AcunetixParser()
+            findings = parser.get_findings(testfile, Test())
+            self.assertEqual(1, len(findings))
+            with self.subTest(i=0):
+                finding = findings[0]
+                # FirstSeenDate is "16/06/2021 12:30 PM"
+                self.assertEqual(finding.date, date(2021, 6, 16, 12, 30))
+
     def test_parse_file_with_one_finding_false_positive(self):
         with (get_unit_tests_scans_path("acunetix") / "acunetix360_one_finding_false_positive.json").open(encoding="utf-8") as testfile:
             parser = AcunetixParser()
@@ -323,11 +339,25 @@ def test_parse_file_issue_10435(self):
             self.assertEqual(1, len(findings))
 
     def test_parse_file_issue_11206(self):
+        """With USE_FIRST_SEEN=False (default), date should come from Generated (scan date)."""
         with (get_unit_tests_scans_path("acunetix") / "issue_11206.json").open(encoding="utf-8") as testfile:
             parser = AcunetixParser()
             findings = parser.get_findings(testfile, Test())
             self.validate_locations(findings)
             self.assertEqual(1, len(findings))
             with self.subTest(i=0):
                 finding = findings[0]
+                # Generated date is "25/06/2021 09:59 AM" — FirstSeenDate ignored when USE_FIRST_SEEN=False
+                self.assertEqual(finding.date, date(2021, 6, 25, 9, 59))
+
+    @override_settings(USE_FIRST_SEEN=True)
+    def test_parse_file_issue_11206_first_seen(self):
+        """With USE_FIRST_SEEN=True, date should come from FirstSeenDate."""
+        with (get_unit_tests_scans_path("acunetix") / "issue_11206.json").open(encoding="utf-8") as testfile:
+            parser = AcunetixParser()
+            findings = parser.get_findings(testfile, Test())
+            self.assertEqual(1, len(findings))
+            with self.subTest(i=0):
+                finding = findings[0]
+                # FirstSeenDate is "12/06/2021 12:30 PM"
                 self.assertEqual(finding.date, date(2021, 6, 12, 12, 30))

unittests/tools/test_checkmarx_parser.py

@@ -2,6 +2,8 @@
 import datetime
 from unittest.mock import patch
 
+from django.test import override_settings
+
 from dojo.models import Engagement, Product, Test
 from dojo.tools.checkmarx.parser import CheckmarxParser
 from unittests.dojo_test_case import DojoTestCase, get_unit_tests_scans_path
@@ -837,6 +839,7 @@ def test_finding_date_should_be_date_xml(self, mock):
 
     @patch("dojo.tools.checkmarx.parser.add_language")
     def test_finding_date_should_be_date_json(self, mock):
+        """With USE_FIRST_SEEN=False (default), date should come from foundDate."""
         my_file_handle, _product, _engagement, test = self.init(
             get_unit_tests_scans_path("checkmarx") / "multiple_findings.json",
         )
@@ -845,3 +848,28 @@ def test_finding_date_should_be_date_json(self, mock):
         findings = parser.get_findings(my_file_handle, test)
         self.teardown(my_file_handle)
         self.assertEqual(findings[0].date, datetime.date(2022, 2, 25))
+
+    @override_settings(USE_FIRST_SEEN=True)
+    @patch("dojo.tools.checkmarx.parser.add_language")
+    def test_finding_date_should_use_first_found_date_json(self, mock):
+        """With USE_FIRST_SEEN=True, date should come from firstFoundDate."""
+        my_file_handle, _product, _engagement, test = self.init(
+            get_unit_tests_scans_path("checkmarx") / "sample_report.json",
+        )
+        parser = CheckmarxParser()
+        findings = parser.get_findings(my_file_handle, test)
+        self.teardown(my_file_handle)
+        # firstFoundDate seconds=1651835169 -> 2022-05-06
+        self.assertEqual(findings[0].date, datetime.date(2022, 5, 6))
+
+    @patch("dojo.tools.checkmarx.parser.add_language")
+    def test_finding_date_should_use_found_date_json(self, mock):
+        """With USE_FIRST_SEEN=False (default), date should come from foundDate."""
+        my_file_handle, _product, _engagement, test = self.init(
+            get_unit_tests_scans_path("checkmarx") / "sample_report.json",
+        )
+        parser = CheckmarxParser()
+        findings = parser.get_findings(my_file_handle, test)
+        self.teardown(my_file_handle)
+        # foundDate seconds=1663153613 -> 2022-09-14
+        self.assertEqual(findings[0].date, datetime.date(2022, 9, 14))

unittests/tools/test_risk_recon_parser.py

@@ -1,6 +1,7 @@
 import datetime
 
 import requests
+from django.test import override_settings
 
 from dojo.models import Test
 from dojo.tools.risk_recon.parser import RiskReconParser
@@ -22,15 +23,32 @@ def test_api_with_bad_key(self):
             parser.get_findings(testfile, Test())
 
     def test_parser_without_api(self):
+        """With USE_FIRST_SEEN=False (default), date should come from last_seen."""
         with (get_unit_tests_scans_path("risk_recon") / "findings.json").open(encoding="utf-8") as testfile:
             parser = RiskReconParser()
             findings = parser.get_findings(testfile, Test())
             self.assertEqual(2, len(findings))
             with self.subTest(i=0):
                 finding = findings[0]
-                self.assertEqual(datetime.date(2017, 3, 17), finding.date.date())
+                # last_seen is "2020-08-13T12:18:45+00:00"
+                self.assertEqual(datetime.date(2020, 8, 13), finding.date.date())
                 self.assertEqual("ff2bbdbfc2b6fddc061ed96b1fasfwefb", finding.unique_id_from_tool)
             with self.subTest(i=1):
                 finding = findings[1]
-                self.assertEqual(datetime.date(2017, 3, 17), finding.date.date())
+                self.assertEqual(datetime.date(2020, 8, 13), finding.date.date())
                 self.assertEqual("ff2bbdbfc2b6gsrgwergwe6b1fasfwefb", finding.unique_id_from_tool)
+
+    @override_settings(USE_FIRST_SEEN=True)
+    def test_parser_without_api_first_seen(self):
+        """With USE_FIRST_SEEN=True, date should come from first_seen."""
+        with (get_unit_tests_scans_path("risk_recon") / "findings.json").open(encoding="utf-8") as testfile:
+            parser = RiskReconParser()
+            findings = parser.get_findings(testfile, Test())
+            self.assertEqual(2, len(findings))
+            with self.subTest(i=0):
+                finding = findings[0]
+                # first_seen is "2017-03-17T00:34:24+00:00"
+                self.assertEqual(datetime.date(2017, 3, 17), finding.date.date())
+            with self.subTest(i=1):
+                finding = findings[1]
+                self.assertEqual(datetime.date(2017, 3, 17), finding.date.date())