➖ Don't collect reference ids in NVD importer

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>

Author: sbs2001

vulnerabilities/importers/nvd.py

@@ -49,7 +49,7 @@ class NVDDataSource(DataSource):
     def updated_advisories(self):
         current_year = date.today().year
         # NVD json feeds start from 2002.
-        for year in range(2002, current_year + 1):
+        for year in range(2002, current_year+1):
             download_url = BASE_URL.format(year)
             # Etags are like hashes of web responses. We maintain
             # (url, etag) mappings in the DB. `create_etag`  creates
@@ -74,7 +74,8 @@ def to_advisories(self, nvd_data):
                 continue
 
             cve_id = cve_item["cve"]["CVE_data_meta"]["ID"]
-            references = self.extract_references(cve_item)
+            ref_urls = self.extract_reference_urls(cve_item)
+            references = [Reference(url=url) for url in ref_urls]
             summary = self.extract_summary(cve_item)
             yield Advisory(
                 cve_id=cve_id, summary=summary, vuln_references=references, impacted_package_urls=[]
@@ -88,31 +89,18 @@ def extract_summary(cve_item):
         summaries = [desc["value"] for desc in cve_item["cve"]["description"]["description_data"]]
         return max(summaries, key=len)
 
-    def extract_references(self, cve_item):
-        refs = []
+    def extract_reference_urls(self, cve_item):
+        urls = set()
         for reference in cve_item["cve"]["references"]["reference_data"]:
-            ref_id = self.find_ref_id(reference)
             ref_url = reference["url"]
 
-            # Skip references which exceed db constraints
-            if ref_id and len(ref_id) > 50:
+            if not ref_url:
                 continue
 
-            refs.append(Reference(url=ref_url, reference_id=ref_id))
+            if ref_url.startswith("http") or ref_url.startswith("ftp"):
+                urls.add(ref_url)
 
-        return refs
-
-    @staticmethod
-    def find_ref_id(reference):
-        if "https://" in reference["name"] or "http://" in reference["name"]:
-            if "bugzilla" in reference["url"]:
-                _, _, bugzilla_id = reference["url"].partition("?id=")
-                return bugzilla_id
-
-            return ""
-
-        else:
-            return reference["name"]
+        return urls
 
     def is_outdated(self, cve_item):
         cve_last_modified_date = cve_item["lastModifiedDate"]

vulnerabilities/tests/test_nvd.py

@@ -87,38 +87,6 @@ def test_extract_summary_with_multiple_summary(self):
         found_summary = NVDDataSource.extract_summary(cve_item)
         assert found_summary == expected_summary
 
-    def test_find_ref_id_when_has_refid(self):
-
-        reference = {
-            "url": "http://www.securityfocus.com/bid/12577",
-            "name": "12577",
-            "refsource": "BID",
-            "tags": [],
-        }
-
-        assert NVDDataSource.find_ref_id(reference) == "12577"
-
-    def test_find_ref_id_when_no_refid(self):
-
-        reference = {
-            "url": "http://www.securityfocus.com/bid/12577",
-            "name": "http://www.securityfocus.com/bid/12577",
-            "refsource": "BID",
-            "tags": [],
-        }
-
-        assert NVDDataSource.find_ref_id(reference) == ""
-
-    def test_find_ref_id_when_has_bugzilla(self):
-
-        reference = {
-            "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=12309",
-            "name": "https://bugzilla.mozilla.org/show_bug.cgi?id=12309",
-            "refsource": "BID",
-            "tags": [],
-        }
-        assert NVDDataSource.find_ref_id(reference) == "12309"
-
     def test_is_outdated(self):
         cve_item = self.nvd_data["CVE_Items"][0]
         assert self.data_src.is_outdated(cve_item) is False
@@ -136,46 +104,24 @@ def test_is_outdated(self):
         assert self.data_src.is_outdated(cve_item) is False
         self.data_src.config.last_run_date = None  # cleanup
 
-    def test_extract_references(self):
-
-        expected_refs = sorted(
-            [
-                Reference(url="http://ia.cr/2007/474", reference_id="2007"),
-                Reference(url="http://shattered.io/", reference_id=""),
-                Reference(
-                    url="http://www.cwi.nl/news/2017/cwi-and-google-announce-first-collision-industry-security-standard-sha-1",  # nopep8
-                    reference_id="",
-                ),
-                Reference(
-                    url="https://arstechnica.com/security/2017/02/at-deaths-door-for-years-widely-used-sha1-function-is-now-dead/",  # nopep8
-                    reference_id="",
-                ),
-                Reference(
-                    url="https://security.googleblog.com/2015/12/an-update-on-sha-1-certificates-in.html",  # nopep8
-                    reference_id="",
-                ),
-                Reference(
-                    url="https://security.googleblog.com/2017/02/announcing-first-sha1-collision.html",  # nopep8
-                    reference_id="",
-                ),
-                Reference(url="https://sites.google.com/site/itstheshappening", reference_id="",),
-                Reference(
-                    url="https://www.schneier.com/blog/archives/2005/02/sha1_broken.html",
-                    reference_id="",
-                ),
-                Reference(
-                    url="https://www.schneier.com/blog/archives/2005/08/new_cryptanalyt.html",
-                    reference_id="",
-                ),
-            ],
-            key=lambda x: x.url,
-        )
-
+    def test_extract_reference_urls(self):
         cve_item = self.nvd_data["CVE_Items"][1]
-        found_refs = self.data_src.extract_references(cve_item)
-        found_refs.sort(key=lambda x: x.url)
+        expected_urls = {
+            "http://ia.cr/2007/474",
+            "http://shattered.io/",
+            "http://www.cwi.nl/news/2017/cwi-and-google-announce-first-collision-industry-security-standard-sha-1",  # nopep8
+            "http://www.securityfocus.com/bid/12577",
+            "https://arstechnica.com/security/2017/02/at-deaths-door-for-years-widely-used-sha1-function-is-now-dead/",  # nopep8
+            "https://security.googleblog.com/2015/12/an-update-on-sha-1-certificates-in.html",
+            "https://security.googleblog.com/2017/02/announcing-first-sha1-collision.html",
+            "https://sites.google.com/site/itstheshappening",
+            "https://www.schneier.com/blog/archives/2005/02/sha1_broken.html",
+            "https://www.schneier.com/blog/archives/2005/08/new_cryptanalyt.html",
+        }
+
+        found_urls = self.data_src.extract_reference_urls(cve_item)
 
-        assert found_refs == expected_refs
+        assert found_urls == expected_urls
 
     def test_to_advisories(self):
 
@@ -193,11 +139,9 @@ def test_to_advisories(self):
                     [
                         Reference(
                             url="http://code.google.com/p/gperftools/source/browse/tags/perftools-0.4/ChangeLog",  # nopep8
-                            reference_id="",
                         ),
                         Reference(
                             url="http://kqueue.org/blog/2012/03/05/memory-allocator-security-revisited/",  # nopep8
-                            reference_id="",
                         ),
                     ],
                     key=lambda x: x.url,