update invicti parser to use FirstSeenDate (#14610)

* update invicti parser to use FirstSeenDate

* invicti: respect USE_FIRST_SEEN

* update invicti test

* ruff

* update invicti test to use Generated scan date

Author: paulOsinski

dojo/tools/netsparker/parser.py

@@ -10,6 +10,25 @@
 from dojo.tools.locations import LocationData
 
 
+def _parse_date(date_str):
+    """Parse a Netsparker/Invicti date string into a date object."""
+    if not date_str:
+        return None
+    try:
+        if "UTC" in date_str:
+            return datetime.datetime.strptime(
+                date_str.split(" ")[0], "%d/%m/%Y",
+            ).date()
+        return datetime.datetime.strptime(
+            date_str, "%d/%m/%Y %H:%M %p",
+        ).date()
+    except ValueError:
+        try:
+            return date_parser.parse(date_str).date()
+        except (ValueError, date_parser.ParserError):
+            return None
+
+
 class NetsparkerParser:
     def get_scan_types(self):
         return ["Netsparker Scan"]
@@ -27,20 +46,7 @@ def get_findings(self, filename, test):
         except Exception:
             data = json.loads(tree)
         dupes = {}
-        try:
-            if "UTC" in data["Generated"]:
-                scan_date = datetime.datetime.strptime(
-                    data["Generated"].split(" ")[0], "%d/%m/%Y",
-                ).date()
-            else:
-                scan_date = datetime.datetime.strptime(
-                    data["Generated"], "%d/%m/%Y %H:%M %p",
-                ).date()
-        except ValueError:
-            try:
-                scan_date = date_parser.parse(data["Generated"])
-            except date_parser.ParserError:
-                scan_date = None
+        scan_date = _parse_date(data.get("Generated"))
 
         for item in data["Vulnerabilities"]:
             title = item["Name"]
@@ -62,6 +68,7 @@ def get_findings(self, filename, test):
             dupe_key = title
             request = item["HttpRequest"].get("Content", None)
             response = item["HttpResponse"].get("Content", None)
+            finding_date = (_parse_date(item.get("FirstSeenDate")) or scan_date) if settings.USE_FIRST_SEEN else scan_date
 
             finding = Finding(
                 title=title,
@@ -70,7 +77,7 @@ def get_findings(self, filename, test):
                 severity=sev.title(),
                 mitigation=mitigation,
                 impact=impact,
-                date=scan_date,
+                date=finding_date,
                 references=references,
                 cwe=cwe,
                 static_finding=True,

unittests/tools/test_netsparker_parser.py

@@ -1,4 +1,6 @@
 
+from django.test import override_settings
+
 from dojo.models import Test
 from dojo.tools.netsparker.parser import NetsparkerParser
 from unittests.dojo_test_case import DojoTestCase, get_unit_tests_scans_path
@@ -7,6 +9,7 @@
 class TestNetsparkerParser(DojoTestCase):
 
     def test_parse_file_with_one_finding(self):
+        """With USE_FIRST_SEEN=False (default), date should come from Generated (scan date)."""
         with (get_unit_tests_scans_path("netsparker") / "netsparker_one_finding.json").open(encoding="utf-8") as testfile:
             parser = NetsparkerParser()
             findings = parser.get_findings(testfile, Test())
@@ -16,6 +19,7 @@ def test_parse_file_with_one_finding(self):
                 finding = findings[0]
                 self.assertEqual("Medium", finding.severity)
                 self.assertEqual(16, finding.cwe)
+                # Generated date is "25/06/2021 09:59 AM"
                 self.assertEqual("25/06/2021", finding.date.strftime("%d/%m/%Y"))
                 self.assertIsNotNone(finding.description)
                 self.assertGreater(len(finding.description), 0)
@@ -24,7 +28,23 @@ def test_parse_file_with_one_finding(self):
                 location = self.get_unsaved_locations(finding)[0]
                 self.assertEqual(str(location), "http://php.testsparker.com/auth/login.php")
 
+    @override_settings(USE_FIRST_SEEN=True)
+    def test_parse_file_with_one_finding_first_seen(self):
+        """With USE_FIRST_SEEN=True, date should come from FirstSeenDate."""
+        with (get_unit_tests_scans_path("netsparker") / "netsparker_one_finding.json").open(encoding="utf-8") as testfile:
+            parser = NetsparkerParser()
+            findings = parser.get_findings(testfile, Test())
+            self.assertEqual(1, len(findings))
+            self.validate_locations(findings)
+            with self.subTest(i=0):
+                finding = findings[0]
+                self.assertEqual("Medium", finding.severity)
+                self.assertEqual(16, finding.cwe)
+                # FirstSeenDate is "16/06/2021 12:30 PM"
+                self.assertEqual("16/06/2021", finding.date.strftime("%d/%m/%Y"))
+
     def test_parse_file_with_multiple_finding(self):
+        """With USE_FIRST_SEEN=False (default), dates should come from Generated (scan date)."""
         with (get_unit_tests_scans_path("netsparker") / "netsparker_many_findings.json").open(encoding="utf-8") as testfile:
             parser = NetsparkerParser()
             findings = parser.get_findings(testfile, Test())
@@ -34,6 +54,7 @@ def test_parse_file_with_multiple_finding(self):
                 finding = findings[0]
                 self.assertEqual("Medium", finding.severity)
                 self.assertEqual(16, finding.cwe)
+                # Generated date is "25/06/2021 10:00 AM"
                 self.assertEqual("25/06/2021", finding.date.strftime("%d/%m/%Y"))
                 self.assertIsNotNone(finding.description)
                 self.assertGreater(len(finding.description), 0)
@@ -66,6 +87,22 @@ def test_parse_file_with_multiple_finding(self):
                 location = self.get_unsaved_locations(finding)[0]
                 self.assertEqual(str(location), "http://php.testsparker.com")
 
+    @override_settings(USE_FIRST_SEEN=True)
+    def test_parse_file_with_multiple_finding_first_seen(self):
+        """With USE_FIRST_SEEN=True, dates should come from FirstSeenDate."""
+        with (get_unit_tests_scans_path("netsparker") / "netsparker_many_findings.json").open(encoding="utf-8") as testfile:
+            parser = NetsparkerParser()
+            findings = parser.get_findings(testfile, Test())
+            self.assertEqual(16, len(findings))
+            with self.subTest(i=0):
+                finding = findings[0]
+                # FirstSeenDate is "16/06/2021 12:30 PM"
+                self.assertEqual("16/06/2021", finding.date.strftime("%d/%m/%Y"))
+            with self.subTest(i=2):
+                finding = findings[2]
+                # FirstSeenDate is "15/06/2021 01:44 PM"
+                self.assertEqual("15/06/2021", finding.date.strftime("%d/%m/%Y"))
+
     def test_parse_file_issue_9816(self):
         with (get_unit_tests_scans_path("netsparker") / "issue_9816.json").open(encoding="utf-8") as testfile:
             parser = NetsparkerParser()
@@ -91,6 +128,7 @@ def test_parse_file_issue_10311(self):
                 self.assertEqual("03/02/2019", finding.date.strftime("%d/%m/%Y"))
 
     def test_parse_file_issue_11020(self):
+        """With USE_FIRST_SEEN=False (default), date should come from Generated (scan date)."""
         with (get_unit_tests_scans_path("netsparker") / "issue_11020.json").open(encoding="utf-8") as testfile:
             parser = NetsparkerParser()
             findings = parser.get_findings(testfile, Test())
@@ -100,4 +138,17 @@ def test_parse_file_issue_11020(self):
                 finding = findings[0]
                 self.assertEqual("Low", finding.severity)
                 self.assertEqual(205, finding.cwe)
+                # Generated date is "2024-10-08 02:33 PM"
                 self.assertEqual("08/10/2024", finding.date.strftime("%d/%m/%Y"))
+
+    @override_settings(USE_FIRST_SEEN=True)
+    def test_parse_file_issue_11020_first_seen(self):
+        """With USE_FIRST_SEEN=True, date should come from FirstSeenDate."""
+        with (get_unit_tests_scans_path("netsparker") / "issue_11020.json").open(encoding="utf-8") as testfile:
+            parser = NetsparkerParser()
+            findings = parser.get_findings(testfile, Test())
+            self.assertEqual(3, len(findings))
+            with self.subTest(i=0):
+                finding = findings[0]
+                # FirstSeenDate is "2024-07-23 05:32 PM"
+                self.assertEqual("23/07/2024", finding.date.strftime("%d/%m/%Y"))