Merge pull request #14626 from valentijnscholten/fix/parser-tags-performance

fix(parsers): use unsaved_tags instead of tags= in Finding constructor

Author: rossops

dojo/tools/anchore_grype/parser.py

@@ -215,16 +215,18 @@ def get_findings(self, file, test):
                     component_name=artifact_name,
                     component_version=artifact_version.replace("\x00", ""),
                     vuln_id_from_tool=vuln_id,
-                    tags=finding_tags,
                     static_finding=True,
                     dynamic_finding=False,
                     nb_occurences=1,
                     file_path=file_path,
                     fix_available=fix_available,
                     fix_version=fix_version,
                 )
+
                 if self.mode == "detailed":
                     dupes[dupe_key].unique_id_from_tool = dupe_key
+
+                dupes[dupe_key].unsaved_tags = finding_tags
                 dupes[dupe_key].unsaved_vulnerability_ids = vulnerability_ids
                 if settings.V3_FEATURE_LOCATIONS and artifact_purl:
                     dupes[dupe_key].unsaved_locations.append(

dojo/tools/cargo_audit/parser.py

@@ -130,7 +130,6 @@ def get_findings(self, filename, test):
                         title=title,
                         test=test,
                         severity=severity,
-                        tags=tags,
                         description=description,
                         component_name=package_name,
                         component_version=package_version,
@@ -140,6 +139,7 @@ def get_findings(self, filename, test):
                         references=references,
                         mitigation=mitigation,
                     )
+                    finding.unsaved_tags = tags
                     finding.unsaved_vulnerability_ids = vulnerability_ids
                     if settings.V3_FEATURE_LOCATIONS and package_name:
                         finding.unsaved_locations.append(

dojo/tools/dependency_check/parser.py

@@ -390,7 +390,6 @@ def get_finding_from_vulnerability(
             mitigation=mitigation,
             mitigated=mitigated,
             is_mitigated=is_Mitigated,
-            tags=tags,
             active=active,
             dynamic_finding=False,
             static_finding=True,
@@ -400,6 +399,8 @@ def get_finding_from_vulnerability(
             **self.get_severity_and_cvss_meta(vulnerability, namespace),
         )
 
+        finding.unsaved_tags = tags
+
         if settings.V3_FEATURE_LOCATIONS and component_purl:
             finding.unsaved_locations.append(
                 LocationData.dependency(purl=component_purl, file_path=dependency_filename),

dojo/tools/jfrog_xray_unified/parser.py

@@ -146,10 +146,11 @@ def get_item(vulnerability, test):
         impact=severity,
         date=scan_time,
         unique_id_from_tool=vulnerability["issue_id"],
-        tags=tags,
         fix_available=fix_available,
     )
 
+    finding.unsaved_tags = tags
+
     cvss_data = parse_cvss_data(cvssv3)
     if cvss_data:
         finding.cvssv3 = cvss_data.get("cvssv3")

dojo/tools/threat_composer/parser.py

@@ -84,11 +84,12 @@ def get_findings(self, file, test):
                     unique_id_from_tool=unique_id_from_tool,
                     mitigation=mitigation,
                     impact=impact,
-                    tags=tags,
                     static_finding=True,
                     dynamic_finding=False,
                 )
 
+                finding.unsaved_tags = tags
+
                 match threat.get("status", "threatIdentified"):
                     case "threatResolved":
                         finding.active = False

unittests/tools/test_anchore_grype_parser.py

@@ -132,7 +132,7 @@ def test_check_all_fields(self):
             self.assertEqual("libgssapi-krb5-2", finding.component_name)
             self.assertEqual("1.17-3+deb10u3", finding.component_version)
             self.assertEqual("CVE-2004-0971", finding.vuln_id_from_tool)
-            self.assertEqual(["dpkg"], finding.tags)
+            self.assertEqual(["dpkg"], finding.unsaved_tags)
             self.assertEqual(1, finding.nb_occurences)
 
             finding = findings[1]
@@ -167,7 +167,7 @@ def test_check_all_fields(self):
             self.assertEqual("redis", finding.component_name)
             self.assertEqual("4.0.2", finding.component_version)
             self.assertEqual("CVE-2021-32626", finding.vuln_id_from_tool)
-            self.assertEqual(["python", "python2"], finding.tags)
+            self.assertEqual(["python", "python2"], finding.unsaved_tags)
             self.assertEqual(1, finding.nb_occurences)
 
             finding = findings[2]
@@ -197,7 +197,7 @@ def test_check_all_fields(self):
             self.assertEqual("libc-bin", finding.component_name)
             self.assertEqual("2.28-10", finding.component_version)
             self.assertEqual("CVE-2021-33574", finding.vuln_id_from_tool)
-            self.assertEqual(["dpkg"], finding.tags)
+            self.assertEqual(["dpkg"], finding.unsaved_tags)
             self.assertEqual(1, finding.nb_occurences)
 
             finding = findings[3]
@@ -227,7 +227,7 @@ def test_check_all_fields(self):
             self.assertEqual("libc6", finding.component_name)
             self.assertEqual("2.28-10", finding.component_version)
             self.assertEqual("CVE-2021-33574", finding.vuln_id_from_tool)
-            self.assertEqual(["dpkg"], finding.tags)
+            self.assertEqual(["dpkg"], finding.unsaved_tags)
             self.assertEqual(1, finding.nb_occurences)
 
             finding = findings[4]
@@ -257,7 +257,7 @@ def test_check_all_fields(self):
             self.assertEqual("Django", finding.component_name)
             self.assertEqual("3.2.9", finding.component_version)
             self.assertEqual("GHSA-v6rh-hp5x-86rv", finding.vuln_id_from_tool)
-            self.assertEqual(["python"], finding.tags)
+            self.assertEqual(["python"], finding.unsaved_tags)
             self.assertEqual(2, finding.nb_occurences)
 
     def test_grype_issue_9618(self):

unittests/tools/test_cargo_audit_parser.py

@@ -22,7 +22,7 @@ def test_parse_many_findings(self):
                 self.assertEqual("[arc-swap 0.4.7] Dangling reference in `access::Map` with Constant", finding.title)
                 self.assertEqual("High", finding.severity)
                 self.assertIsNotNone(finding.description)
-                self.assertEqual(["dangling reference"], finding.tags)
+                self.assertEqual(["dangling reference"], finding.unsaved_tags)
                 self.assertEqual("arc-swap", finding.component_name)
                 self.assertEqual("0.4.7", finding.component_version)
                 self.assertEqual("RUSTSEC-2020-0091", finding.vuln_id_from_tool)
@@ -37,7 +37,7 @@ def test_parse_many_findings(self):
                 self.assertEqual("[hyper 0.13.9] Multiple Transfer-Encoding headers misinterprets request payload", finding.title)
                 self.assertEqual("High", finding.severity)
                 self.assertIsNotNone(finding.description)
-                self.assertEqual(["http", "request-smuggling"], finding.tags)
+                self.assertEqual(["http", "request-smuggling"], finding.unsaved_tags)
                 self.assertEqual("hyper", finding.component_name)
                 self.assertEqual("0.13.9", finding.component_version)
                 self.assertEqual("RUSTSEC-2021-0020", finding.vuln_id_from_tool)
@@ -52,7 +52,7 @@ def test_parse_many_findings(self):
                 self.assertEqual("[smallvec 0.6.13] Buffer overflow in SmallVec::insert_many", finding.title)
                 self.assertEqual("High", finding.severity)
                 self.assertIsNotNone(finding.description)
-                self.assertEqual(["buffer-overflow", "heap-overflow", "unsound"], finding.tags)
+                self.assertEqual(["buffer-overflow", "heap-overflow", "unsound"], finding.unsaved_tags)
                 self.assertEqual("smallvec", finding.component_name)
                 self.assertEqual("0.6.13", finding.component_version)
                 self.assertEqual("RUSTSEC-2021-0003", finding.vuln_id_from_tool)
@@ -67,7 +67,7 @@ def test_parse_many_findings(self):
                 self.assertEqual("[smallvec 1.5.0] Buffer overflow in SmallVec::insert_many", finding.title)
                 self.assertEqual("High", finding.severity)
                 self.assertIsNotNone(finding.description)
-                self.assertEqual(["buffer-overflow", "heap-overflow", "unsound"], finding.tags)
+                self.assertEqual(["buffer-overflow", "heap-overflow", "unsound"], finding.unsaved_tags)
                 self.assertEqual("smallvec", finding.component_name)
                 self.assertEqual("1.5.0", finding.component_version)
                 self.assertEqual("RUSTSEC-2021-0003", finding.vuln_id_from_tool)

unittests/tools/test_dependency_check_parser.py

@@ -108,7 +108,7 @@ def test_parse_file_with_multiple_vulnerabilities_has_multiple_findings(self):
                     items[1].mitigation,
                     "Update org.dom4j:dom4j:2.1.1.redhat-00001 to at least the version recommended in the description",
                 )
-                self.assertEqual(items[1].tags, "related")
+                self.assertEqual(items[1].unsaved_tags, ["related"])
                 self.assertEqual(1, len(items[1].unsaved_vulnerability_ids))
                 self.assertEqual("CVE-0000-0001", items[1].unsaved_vulnerability_ids[0])
 
@@ -258,7 +258,7 @@ def test_parse_file_with_multiple_vulnerabilities_has_multiple_findings(self):
                     items[9].mitigation,
                     "**This vulnerability is mitigated and/or suppressed:** Document on why we are suppressing this vulnerability is missing!\nUpdate jquery:3.1.1 to at least the version recommended in the description",
                 )
-                self.assertEqual(items[9].tags, ["suppressed", "no_suppression_document"])
+                self.assertEqual(items[9].unsaved_tags, ["no_suppression_document", "suppressed"])
                 self.assertEqual(items[9].severity, "Critical")
                 self.assertEqual(items[9].cvssv3, "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H")
                 self.assertEqual(items[9].cvssv3_score, 9.8)
@@ -270,7 +270,7 @@ def test_parse_file_with_multiple_vulnerabilities_has_multiple_findings(self):
                     items[10].mitigation,
                     "**This vulnerability is mitigated and/or suppressed:** This is our reason for not to upgrade it.\nUpdate jquery:3.1.1 to at least the version recommended in the description",
                 )
-                self.assertEqual(items[10].tags, "suppressed")
+                self.assertEqual(items[10].unsaved_tags, ["suppressed"])
                 self.assertEqual(items[10].severity, "Critical")
                 self.assertEqual(items[10].cvssv3, "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H")
                 self.assertEqual(items[10].cvssv3_score, 9.8)

unittests/tools/test_jfrog_xray_unified_parser.py

@@ -33,7 +33,7 @@ def test_parse_file_with_one_vuln(self):
         self.assertIsNotNone(item.mitigation)
         self.assertGreater(len(item.mitigation), 0)
         self.assertEqual("Jinja2", item.component_name)
-        self.assertEqual('"packagetype_pypi"', item.tags)
+        self.assertEqual(["packagetype_pypi"], item.unsaved_tags)
         self.assertEqual("2.11.2", item.component_version)
         self.assertEqual("pypi-remote/30/9e/f663a2aa66a09d838042ae1a2c5659828bb9b41ea3a6efa20a20fd92b121/Jinja2-2.11.2-py2.py3-none-any.whl", item.file_path)
         self.assertIsNotNone(item.severity_justification)
@@ -186,7 +186,7 @@ def test_parse_file_with_very_many_vulns(self):
         self.assertEqual(" is too late.", item.description[-13:])
         self.assertIsNone(item.mitigation)
         self.assertEqual("3.12:sqlite-libs", item.component_name)
-        self.assertEqual('"packagetype_alpine"', item.tags)
+        self.assertEqual(["packagetype_alpine"], item.unsaved_tags)
         self.assertEqual("3.32.1-r0", item.component_version)
         self.assertEqual("dockerhub-remote/kiwigrid/k8s-sidecar/sha256__7cba93c3dde21c78fe07ee3f8ed8d82d05bf00415392606401df8a7d72057b5b/", item.file_path)
         self.assertIsNotNone(item.severity_justification)
@@ -209,7 +209,7 @@ def test_parse_file_with_very_many_vulns(self):
         self.assertEqual("(Affected 1.0.2-1.0.2w).", item.description[-24:])
         self.assertIsNone(item.mitigation)
         self.assertEqual("ubuntu:bionic:libssl1.1", item.component_name)
-        self.assertEqual('"packagetype_debian"', item.tags)
+        self.assertEqual(["packagetype_debian"], item.unsaved_tags)
         self.assertEqual("1.1.1-1ubuntu2.1~18.04.6", item.component_version)
         self.assertEqual("dockerhub-remote/library/mongo/sha256__31f6433f7cfcd2180483e40728cbf97142df1e85de36d80d75c93e5e7fe10405/", item.file_path)
         self.assertIsNotNone(item.severity_justification)
@@ -233,7 +233,7 @@ def test_parse_file_with_very_many_vulns(self):
         self.assertIsNotNone(item.mitigation)
         self.assertGreater(len(item.mitigation), 0)
         self.assertEqual("github.com/docker/docker", item.component_name)
-        self.assertEqual('"packagetype_go"', item.tags)
+        self.assertEqual(["packagetype_go"], item.unsaved_tags)
         self.assertEqual("1.4.2-0.20200203170920-46ec8731fbce", item.component_version)
         self.assertEqual("dockerhub-remote/fluxcd/helm-controller/sha256__27790f965d8965884e8dfc12cba0d1f609794a1abc69bc81a658bd76e463ffce/", item.file_path)
         self.assertIsNotNone(item.severity_justification)
@@ -255,7 +255,7 @@ def test_parse_file_with_very_many_vulns(self):
         self.assertEqual("sensitive information.", item.description[-22:])
         self.assertIsNone(item.mitigation)
         self.assertEqual("com.fasterxml.jackson.core:jackson-databind", item.component_name)
-        self.assertEqual('"packagetype_maven"', item.tags)
+        self.assertEqual(["packagetype_maven"], item.unsaved_tags)
         self.assertEqual("2.10.4", item.component_version)
         self.assertEqual("elastic-docker-remote/elasticsearch/elasticsearch/7.9.1-amd64/", item.file_path)
         self.assertIsNotNone(item.severity_justification)
@@ -279,7 +279,7 @@ def test_parse_file_with_very_many_vulns(self):
         self.assertIsNotNone(item.mitigation)
         self.assertGreater(len(item.mitigation), 0)
         self.assertEqual("jquery", item.component_name)
-        self.assertEqual('"packagetype_npm"', item.tags)
+        self.assertEqual(["packagetype_npm"], item.unsaved_tags)
         self.assertEqual("3.4.1", item.component_version)
         self.assertEqual("pypi-remote/cc/94/5f7079a0e00bd6863ef8f1da638721e9da21e5bacee597595b318f71d62e/Werkzeug-1.0.1-py2.py3-none-any.whl", item.file_path)
         self.assertIsNotNone(item.severity_justification)
@@ -303,7 +303,7 @@ def test_parse_file_with_very_many_vulns(self):
         self.assertIsNotNone(item.mitigation)
         self.assertGreater(len(item.mitigation), 0)
         self.assertEqual("pip", item.component_name)
-        self.assertEqual('"packagetype_pypi"', item.tags)
+        self.assertEqual(["packagetype_pypi"], item.unsaved_tags)
         self.assertEqual("20.2.3", item.component_version)
         self.assertEqual("dockerhub-remote/kiwigrid/k8s-sidecar/sha256__4b5a25c8dbac9637f8e680566959fdccd1a98d74ce2f2746f9b0f9ff6b57d03b/", item.file_path)
         self.assertIsNotNone(item.severity_justification)
@@ -326,7 +326,7 @@ def test_parse_file_with_very_many_vulns(self):
         self.assertEqual("TABLE statements.\n\nRed Hat Severity: Moderate", item.description[-45:])
         self.assertIsNone(item.mitigation)
         self.assertEqual("7:sqlite:0", item.component_name)
-        self.assertIn("packagetype_rpm", item.tags)
+        self.assertIn("packagetype_rpm", item.unsaved_tags)
         self.assertEqual("3.7.17-8.el7_7.1", item.component_version)
         self.assertEqual("elastic-docker-remote/elasticsearch/elasticsearch/7.9.1-amd64/", item.file_path)
         self.assertIsNotNone(item.severity_justification)