Review all V2 pipelines

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

Author: TG1999

vulnerabilities/importers/fireeye.py

@@ -112,7 +112,7 @@ def matcher_url(ref) -> str:
     """
     Returns URL of the reference markup from reference url in Markdown format
     """
-    markup_regex = "\[([^\[]+)]\(\s*(http[s]?://.+)\s*\)"
+    markup_regex = r"\[([^\[]+)]\(\s*(http[s]?://.+)\s*\)"
     matched_markup = re.findall(markup_regex, ref)
     if matched_markup:
         return matched_markup[0][1]

vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py

@@ -7,6 +7,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import json
 import logging
 from pathlib import Path
 from typing import Any
@@ -244,4 +245,5 @@ def load_advisories(
                 references=references,
                 affected_packages=affected_packages,
                 url=url,
+                original_advisory_text=json.dumps(pkg_infos, indent=2, ensure_ascii=False),
             )

vulnerabilities/pipelines/v2_importers/apache_tomcat_importer.py

@@ -111,6 +111,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
                         summary=advisory_list[0].summary,
                         affected_packages=affected_packages,
                         url=page_url,
+                        original_advisory_text=str(content),
                     )
 
             except Exception as e:

vulnerabilities/pipelines/v2_importers/archlinux_importer.py

@@ -14,10 +14,13 @@
 from packageurl import PackageURL
 from univers.version_range import ArchLinuxVersionRange
 
+from vulnerabilities import severity_systems
 from vulnerabilities.importer import AdvisoryDataV2
 from vulnerabilities.importer import AffectedPackageV2
 from vulnerabilities.importer import ReferenceV2
+from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+from vulnerabilities.severity_systems import SCORING_SYSTEMS
 from vulnerabilities.utils import fetch_response
 
 
@@ -53,7 +56,9 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
     def parse_advisory(self, record) -> AdvisoryDataV2:
         affected_packages = []
         references = []
+        severities = []
         avg_name = record.get("name")
+        severity = record.get("severity")
         aliases = record.get("issues", [])
         aliases.extend(record.get("advisories", []))
         summary = record.get("type", "")
@@ -92,13 +97,26 @@ def parse_advisory(self, record) -> AdvisoryDataV2:
                 )
             )
 
+        if severity not in severity_systems.ARCHLINUX.choices:
+            self.log(f"Unknown severity {severity} for {avg_name}")
+            severity = None
+        if severity:
+            severities = [
+                VulnerabilitySeverity(
+                    system=severity_systems.ARCHLINUX,
+                    value=severity,
+                    url="https://security.archlinux.org/{avg_name}.json",
+                )
+            ]
+
         return AdvisoryDataV2(
             advisory_id=avg_name,
             aliases=aliases,
             summary=summary,
             references=references,
             affected_packages=affected_packages,
+            severities=severities,
             weaknesses=[],
             url=f"https://security.archlinux.org/{avg_name}.json",
-            original_advisory_text=json.dumps(record),
+            original_advisory_text=json.dumps(record, indent=2, ensure_ascii=False),
         )

vulnerabilities/pipelines/v2_importers/debian_importer.py

@@ -7,6 +7,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import json
 import re
 from typing import Any
 from typing import Iterable
@@ -171,6 +172,7 @@ def parse(self, pkg_name: str, records: Mapping[str, Any]) -> Iterable[AdvisoryD
                 references=references,
                 weaknesses=weaknesses,
                 url=f"https://security-tracker.debian.org/tracker/{record_identifier}",
+                original_advisory_text=json.dumps(record, indent=2, ensure_ascii=False),
             )
 
 

vulnerabilities/pipelines/v2_importers/epss_importer_v2.py

@@ -83,4 +83,5 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
                 severities=[severity],
                 references=[references],
                 url=self.advisory_url,
+                original_advisory_text=",".join(epss_row),
             )

vulnerabilities/pipelines/v2_importers/fireeye_importer_v2.py

@@ -154,7 +154,7 @@ def matcher_url(ref) -> str:
     """
     Returns URL of the reference markup from reference url in Markdown format
     """
-    markup_regex = "\[([^\[]+)]\(\s*(http[s]?://.+)\s*\)"
+    markup_regex = r"\[([^\[]+)]\(\s*(http[s]?://.+)\s*\)"
     matched_markup = re.findall(markup_regex, ref)
     if matched_markup:
         return matched_markup[0][1]

vulnerabilities/pipelines/v2_importers/istio_importer.py

@@ -42,6 +42,7 @@ class IstioImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     spdx_license_expression = "Apache-2.0"
     license_url = "https://github.com/istio/istio.io/blob/master/LICENSE"
     repo_url = "git+https://github.com/istio/istio.io"
+    run_once = True
 
     precedence = 200
 

vulnerabilities/pipelines/v2_importers/mattermost_importer.py

@@ -7,6 +7,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import json
 from typing import Iterable
 
 from packageurl import PackageURL
@@ -122,5 +123,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
                 summary=details,
                 references=[reference],
                 affected_packages=affected_packages,
+                severities=severities,
                 url=self.url,
+                original_advisory_text=json.dumps(advisory, indent=2, ensure_ascii=False),
             )

vulnerabilities/pipelines/v2_importers/nginx_importer.py

@@ -62,7 +62,7 @@ def collect_advisories(self):
         vulnerability_list = soup.select("li p")
         for vulnerability_info in vulnerability_list:
             ngnix_advisory = parse_advisory_data_from_paragraph(vulnerability_info)
-            yield to_advisory_data(ngnix_advisory)
+            yield to_advisory_data(ngnix_advisory, vulnerability_info)
 
 
 class NginxAdvisory(NamedTuple):
@@ -79,7 +79,7 @@ def to_dict(self):
         return self._asdict()
 
 
-def to_advisory_data(nginx_adv: NginxAdvisory) -> AdvisoryDataV2:
+def to_advisory_data(nginx_adv: NginxAdvisory, vulnerability_info) -> AdvisoryDataV2:
     """
     Return AdvisoryDataV2 from an NginxAdvisory tuple.
     """
@@ -150,6 +150,7 @@ def to_advisory_data(nginx_adv: NginxAdvisory) -> AdvisoryDataV2:
         references=nginx_adv.references,
         patches=nginx_adv.patches,
         url="https://nginx.org/en/security_advisories.html",
+        original_advisory_text=str(vulnerability_info),
     )