Split the project-kb into two separate pipelines

Update Project-KB importer

Signed-off-by: ziad hany <ziadhany2016@gmail.com>

Author: ziadhany

vulnerabilities/importers/__init__.py

@@ -24,6 +24,7 @@
 from vulnerabilities.importers import openssl
 from vulnerabilities.importers import oss_fuzz
 from vulnerabilities.importers import postgresql
+from vulnerabilities.importers import project_kb_msr2019
 from vulnerabilities.importers import redhat
 from vulnerabilities.importers import retiredotnet
 from vulnerabilities.importers import ruby
@@ -58,7 +59,12 @@
 from vulnerabilities.pipelines.v2_importers import nvd_importer as nvd_importer_v2
 from vulnerabilities.pipelines.v2_importers import oss_fuzz as oss_fuzz_v2
 from vulnerabilities.pipelines.v2_importers import postgresql_importer as postgresql_importer_v2
-from vulnerabilities.pipelines.v2_importers import project_kb_importer as project_kb_importer_v2
+from vulnerabilities.pipelines.v2_importers import (
+    project_kb_msr2019_importer as project_kb_msr2019_importer_v2,
+)
+from vulnerabilities.pipelines.v2_importers import (
+    project_kb_statements_importer as project_kb_statements_importer_v2,
+)
 from vulnerabilities.pipelines.v2_importers import pypa_importer as pypa_importer_v2
 from vulnerabilities.pipelines.v2_importers import pysec_importer as pysec_importer_v2
 from vulnerabilities.pipelines.v2_importers import redhat_importer as redhat_importer_v2
@@ -87,7 +93,8 @@
         github_osv_importer_v2.GithubOSVImporterPipeline,
         redhat_importer_v2.RedHatImporterPipeline,
         aosp_importer_v2.AospImporterPipeline,
-        project_kb_importer_v2.ProjectKBPipeline,
+        project_kb_statements_importer_v2.ProjectKBStatementsPipeline,
+        project_kb_msr2019_importer_v2.ProjectKBMSR2019Pipeline,
         ruby_importer_v2.RubyImporterPipeline,
         epss_importer_v2.EPSSImporterPipeline,
         mattermost_importer_v2.MattermostImporterPipeline,
@@ -119,6 +126,7 @@
         mozilla.MozillaImporter,
         gentoo.GentooImporter,
         istio.IstioImporter,
+        project_kb_msr2019.ProjectKBMSRImporter,
         suse_scores.SUSESeverityScoreImporter,
         elixir_security.ElixirSecurityImporter,
         xen.XenImporter,

vulnerabilities/importers/project_kb_msr2019.py

@@ -0,0 +1,46 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import Importer
+from vulnerabilities.importer import Reference
+from vulnerabilities.utils import fetch_and_read_from_csv
+from vulnerabilities.utils import is_cve
+
+# Reading CSV file from  a url using `requests` is bit too complicated.
+# Use `urllib.request` for that purpose.
+
+
+class ProjectKBMSRImporter(Importer):
+
+    url = "https://raw.githubusercontent.com/SAP/project-kb/master/MSR2019/dataset/vulas_db_msr2019_release.csv"
+    spdx_license_expression = "Apache-2.0"
+    license_url = "https://github.com/SAP/project-kb/blob/main/LICENSE.txt"
+    importer_name = "ProjectKB MSRImporter"
+
+    def advisory_data(self):
+        raw_data = fetch_and_read_from_csv(self.url)
+        yield from self.to_advisories(raw_data)
+
+    def to_advisories(self, csv_reader):
+        # Project KB MSR csv file has no header row
+        for row in csv_reader:
+            vuln_id, proj_home, fix_commit, _ = row
+            commit_link = proj_home + "/commit/" + fix_commit
+
+            if not is_cve(vuln_id):
+                continue
+
+            reference = Reference(url=commit_link)
+            yield AdvisoryData(
+                aliases=[vuln_id],
+                summary="",
+                references=[reference],
+                url=self.url,
+            )

vulnerabilities/pipelines/v2_importers/project_kb_msr2019_importer.py

@@ -0,0 +1,93 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import csv
+from pathlib import Path
+from typing import Iterable
+
+from fetchcode.vcs import fetch_via_vcs
+
+from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+from vulnerabilities.pipes.advisory import append_patch_classifications
+
+
+class ProjectKBMSR2019Pipeline(VulnerableCodeBaseImporterPipelineV2):
+    """
+    ProjectKB Importer Pipeline
+    Collect advisory from ProjectKB data:
+    - CSV database https://github.com/SAP/project-kb/blob/main/MSR2019/dataset/vulas_db_msr2019_release.csv
+    """
+
+    pipeline_id = "project-kb-MSR-2019_v2"
+    spdx_license_expression = "Apache-2.0"
+    license_url = "https://github.com/SAP/project-kb/blob/main/LICENSE.txt"
+    repo_url = "git+https://github.com/SAP/project-kb"
+
+    @classmethod
+    def steps(cls):
+        return (cls.clone_repo, cls.collect_and_store_advisories, cls.clean_downloads)
+
+    def clone_repo(self):
+        self.log("Cloning ProjectKB advisory data...")
+        self.vcs_response = fetch_via_vcs(self.repo_url)
+
+    def advisories_count(self):
+        csv_path = Path(self.vcs_response.dest_dir) / "MSR2019/dataset/vulas_db_msr2019_release.csv"
+
+        with open(csv_path, newline="", encoding="utf-8") as f:
+            reader = csv.reader(f)
+            next(reader, None)
+            count = sum(1 for _ in reader)
+
+        self.log(f"Estimated advisories to process: {count}")
+        return count
+
+    def collect_advisories(self) -> Iterable[AdvisoryData]:
+        self.log("Collecting fix commits from ProjectKB ( vulas_db_msr2019_release )...")
+        csv_path = Path(self.vcs_response.dest_dir) / "MSR2019/dataset/vulas_db_msr2019_release.csv"
+
+        with open(csv_path, newline="", encoding="utf-8") as f:
+            reader = csv.reader(f)
+            next(reader, None)  # skip header
+            rows = [r for r in reader if len(r) == 4 and r[0]]  # vuln_id, vcs_url, commit_hash, poc
+
+        for vuln_id, vcs_url, commit_hash, _ in rows:
+            if not vuln_id or not vcs_url or not commit_hash:
+                continue
+
+            patches = []
+            affected_packages = []
+            references = []
+            append_patch_classifications(
+                url=vcs_url,
+                commit_hash=commit_hash,
+                patch_text=None,
+                affected_packages=affected_packages,
+                references=references,
+                patches=patches,
+            )
+
+            yield AdvisoryData(
+                advisory_id=vuln_id,
+                affected_packages=affected_packages,
+                patches=patches,
+                references_v2=references,
+                url="https://github.com/SAP/project-kb/blob/main/MSR2019/dataset/vulas_db_msr2019_release.csv",
+            )
+
+    def clean_downloads(self):
+        """Remove the cloned repository from disk."""
+        self.log("Removing cloned repository...")
+        if self.vcs_response:
+            self.vcs_response.delete()
+
+    def on_failure(self):
+        """Ensure cleanup happens on pipeline failure."""
+        self.clean_downloads()