Merge pull request #2080 from ziadhany/osv-migration

Fix OSV to handle affected_packages correctly & add support to collect commits

Author: ziadhany

vulnerabilities/pipelines/v2_importers/github_osv_importer.py

@@ -15,6 +15,7 @@
 
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+from vulnerabilities.pipes.osv_v2 import parse_advisory_data_v3
 from vulnerabilities.utils import get_advisory_url
 
 
@@ -47,8 +48,6 @@ def advisories_count(self):
         return sum(1 for _ in advisory_dir.rglob("*.json"))
 
     def collect_advisories(self) -> Iterable[AdvisoryData]:
-        from vulnerabilities.importers.osv import parse_advisory_data_v2
-
         supported_ecosystems = [
             "pypi",
             "npm",
@@ -72,7 +71,7 @@ def collect_advisories(self) -> Iterable[AdvisoryData]:
             with open(file) as f:
                 raw_data = json.load(f)
             advisory_text = file.read_text()
-            yield parse_advisory_data_v2(
+            yield parse_advisory_data_v3(
                 raw_data=raw_data,
                 supported_ecosystems=supported_ecosystems,
                 advisory_url=advisory_url,

vulnerabilities/pipelines/v2_importers/oss_fuzz.py

@@ -15,6 +15,7 @@
 
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+from vulnerabilities.pipes.osv_v2 import parse_advisory_data_v3
 from vulnerabilities.utils import get_advisory_url
 
 logger = logging.getLogger(__name__)
@@ -43,8 +44,6 @@ def advisories_count(self):
         return sum(1 for _ in vulns_directory.rglob("*.yaml"))
 
     def collect_advisories(self) -> Iterable[AdvisoryData]:
-        from vulnerabilities.importers.osv import parse_advisory_data_v2
-
         base_directory = Path(self.vcs_response.dest_dir)
         vulns_directory = base_directory / "vulns"
 
@@ -56,7 +55,7 @@ def collect_advisories(self) -> Iterable[AdvisoryData]:
             )
             advisory_text = advisory.read_text()
             advisory_dict = saneyaml.load(advisory_text)
-            yield parse_advisory_data_v2(
+            yield parse_advisory_data_v3(
                 raw_data=advisory_dict,
                 supported_ecosystems=["generic"],
                 advisory_url=advisory_url,

vulnerabilities/pipelines/v2_importers/pypa_importer.py

@@ -15,6 +15,7 @@
 
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+from vulnerabilities.pipes.osv_v2 import parse_advisory_data_v3
 from vulnerabilities.utils import get_advisory_url
 
 
@@ -46,8 +47,6 @@ def advisories_count(self):
         return sum(1 for _ in vulns_directory.rglob("*.yaml"))
 
     def collect_advisories(self) -> Iterable[AdvisoryData]:
-        from vulnerabilities.importers.osv import parse_advisory_data_v2
-
         base_directory = Path(self.vcs_response.dest_dir)
         vulns_directory = base_directory / "vulns"
 
@@ -59,7 +58,7 @@ def collect_advisories(self) -> Iterable[AdvisoryData]:
             )
             advisory_text = advisory.read_text()
             advisory_dict = saneyaml.load(advisory_text)
-            yield parse_advisory_data_v2(
+            yield parse_advisory_data_v3(
                 raw_data=advisory_dict,
                 supported_ecosystems=["pypi"],
                 advisory_url=advisory_url,

vulnerabilities/pipelines/v2_importers/pysec_importer.py

@@ -16,6 +16,7 @@
 
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+from vulnerabilities.pipes.osv_v2 import parse_advisory_data_v3
 
 
 class PyPIImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
@@ -47,7 +48,6 @@ def advisories_count(self) -> int:
 
     def collect_advisories(self) -> Iterable[AdvisoryData]:
         """Yield AdvisoryData using a zipped data dump of OSV data"""
-        from vulnerabilities.importers.osv import parse_advisory_data_v2
 
         with ZipFile(BytesIO(self.advisory_zip)) as zip_file:
             for file_name in zip_file.namelist():
@@ -60,7 +60,7 @@ def collect_advisories(self) -> Iterable[AdvisoryData]:
                 with zip_file.open(file_name) as f:
                     vul_info = json.load(f)
                     advisory_text = f.read()
-                    yield parse_advisory_data_v2(
+                    yield parse_advisory_data_v3(
                         raw_data=vul_info,
                         supported_ecosystems=["pypi"],
                         advisory_url=self.url,