Update collect_issue_pr_v2 to use the new AdvisoryDataV2 and refactor the code

Signed-off-by: ziad hany <ziadhany2016@gmail.com>

Author: ziadhany

vulnerabilities/importers/__init__.py

@@ -47,16 +47,15 @@
 from vulnerabilities.pipelines.v2_importers import apache_kafka_importer as apache_kafka_importer_v2
 from vulnerabilities.pipelines.v2_importers import apache_tomcat_importer as apache_tomcat_v2
 from vulnerabilities.pipelines.v2_importers import archlinux_importer as archlinux_importer_v2
-from vulnerabilities.pipelines.v2_importers import collect_issue_pr as collect_issue_pr_v2
 from vulnerabilities.pipelines.v2_importers import collect_fix_commits as collect_fix_commits_v2
+from vulnerabilities.pipelines.v2_importers import collect_issue_pr as collect_issue_pr_v2
 from vulnerabilities.pipelines.v2_importers import curl_importer as curl_importer_v2
 from vulnerabilities.pipelines.v2_importers import debian_importer as debian_importer_v2
 from vulnerabilities.pipelines.v2_importers import (
     elixir_security_importer as elixir_security_importer_v2,
 )
 from vulnerabilities.pipelines.v2_importers import epss_importer_v2
 from vulnerabilities.pipelines.v2_importers import fireeye_importer_v2
-from vulnerabilities.pipelines.v2_importers import github_issue_pr as github_issue_pr_v2
 from vulnerabilities.pipelines.v2_importers import gentoo_importer as gentoo_importer_v2
 from vulnerabilities.pipelines.v2_importers import github_osv_importer as github_osv_importer_v2
 from vulnerabilities.pipelines.v2_importers import gitlab_importer as gitlab_importer_v2
@@ -115,8 +114,6 @@
         nginx_importer_v2.NginxImporterPipeline,
         debian_importer_v2.DebianImporterPipeline,
         mattermost_importer_v2.MattermostImporterPipeline,
-        collect_issue_pr_v2.CollectIssuePRPipeline,
-        github_issue_pr_v2.GithubPipelineIssuePRPipeline,
         apache_tomcat_v2.ApacheTomcatImporterPipeline,
         suse_score_importer_v2.SUSESeverityScoreImporterPipeline,
         retiredotnet_importer_v2.RetireDotnetImporterPipeline,
@@ -158,9 +155,6 @@
         ubuntu_usn.UbuntuUSNImporter,
         fireeye.FireyeImporter,
         oss_fuzz.OSSFuzzImporter,
-        collect_issue_pr_v2.CollectKubernetesPRSIssues,
-        collect_issue_pr_v2.CollectWiresharkPRSIssues,
-        github_issue_pr_v2.GithubPipelineIssuePR,
         collect_fix_commits_v2.CollectLinuxFixCommitsPipeline,
         collect_fix_commits_v2.CollectBusyBoxFixCommitsPipeline,
         collect_fix_commits_v2.CollectNginxFixCommitsPipeline,
@@ -198,5 +192,37 @@
         collect_fix_commits_v2.CollectGitFixCommitsPipeline,
         collect_fix_commits_v2.CollectJenkinsFixCommitsPipeline,
         collect_fix_commits_v2.CollectGitlabFixCommitsPipeline,
+        collect_issue_pr_v2.CollectBusyBoxPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectNginxPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectApacheTomcatPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectMongodbPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectRedisPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectPhpPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectPythonCpythonPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectRubyPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectGoPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectNodeJsPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectRustPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectOpenjdkPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectSwiftPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectDjangoPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectLaravelPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectSpringFrameworkPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectRailsPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectReactPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectAngularPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectWordpressPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectDockerMobyPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectKubernetesPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectXenProjectPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectVirtualboxPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectContainerdPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectAnsiblePRSIssuesPipeline,
+        collect_issue_pr_v2.CollectTerraformPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectTcpdumpPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectJenkinsPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectGitlabPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectWiresharkPRSIssuesPipeline,
+        collect_issue_pr_v2.CollectQemuPRSIssuesPipeline,
     ]
 )

vulnerabilities/pipelines/__init__.py

@@ -25,8 +25,11 @@
 from aboutcode.pipeline import PipelineDefinition
 from aboutcode.pipeline import humanize_time
 from github import Github
+from gitlab import GitlabAuthenticationError
+from gitlab import GitlabSearchError
 
 from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import AdvisoryDataV2
 from vulnerabilities.importer import ReferenceV2
 from vulnerabilities.improver import MAX_CONFIDENCE
 from vulnerabilities.models import Advisory
@@ -389,12 +392,11 @@ def collect_advisories(self):
         """
         self.log("Generating AdvisoryData objects from GitHub/Gitlab issues and PRs.")
         for vuln_id, refs in self.collected_items.items():
-            print(vuln_id, refs)
             references = [ReferenceV2(reference_type=ref_id, url=url) for ref_id, url in refs]
-            yield AdvisoryData(
+            yield AdvisoryDataV2(
                 advisory_id=vuln_id,
                 aliases=[],
-                references_v2=references,
+                references=references,
                 url=self.repo_url,
             )
 

vulnerabilities/pipelines/v2_importers/collect_issue_pr.py

@@ -6,15 +6,166 @@
 # See https://github.com/aboutcode-org/vulnerablecode for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
+
 from vulnerabilities.pipelines import GitHubCollector
 from vulnerabilities.pipelines import GitLabCollector
 
 
-class CollectKubernetesPRSIssues(GitHubCollector):
-    pipeline_id = "collect-kubernetes-prs-issues"
+class CollectBusyBoxPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_busybox_prs-issues"
+    repo_url = "https://github.com/mirror/busybox"
+
+
+class CollectNginxPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_nginx_prs-issues"
+    repo_url = "https://github.com/nginx/nginx"
+
+
+class CollectApacheTomcatPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_apache_tomcat_prs-issues"
+    repo_url = "https://github.com/apache/tomcat"
+
+
+class CollectMongodbPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_mongodb_prs-issues"
+    repo_url = "https://github.com/mongodb/mongo"
+
+
+class CollectRedisPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_redis_prs-issues"
+    repo_url = "https://github.com/redis/redis"
+
+
+class CollectPhpPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_php_prs-issues"
+    repo_url = "https://github.com/php/php-src"
+
+
+class CollectPythonCpythonPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_python_cpython_prs-issues"
+    repo_url = "https://github.com/python/cpython"
+
+
+class CollectRubyPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_ruby_prs-issues"
+    repo_url = "https://github.com/ruby/ruby"
+
+
+class CollectGoPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_go_prs-issues"
+    repo_url = "https://github.com/golang/go"
+
+
+class CollectNodeJsPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_node_js_prs-issues"
+    repo_url = "https://github.com/nodejs/node"
+
+
+class CollectRustPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_rust_prs-issues"
+    repo_url = "https://github.com/rust-lang/rust"
+
+
+class CollectOpenjdkPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_openjdk_prs-issues"
+    repo_url = "https://github.com/openjdk/jdk"
+
+
+class CollectSwiftPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_swift_prs-issues"
+    repo_url = "https://github.com/swiftlang/swift"
+
+
+class CollectDjangoPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_django_prs-issues"
+    repo_url = "https://github.com/django/django"
+
+
+class CollectRailsPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_rails_prs-issues"
+    repo_url = "https://github.com/rails/rails"
+
+
+class CollectLaravelPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_laravel_prs-issues"
+    repo_url = "https://github.com/laravel/framework"
+
+
+class CollectSpringFrameworkPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_spring_framework_prs-issues"
+    repo_url = "https://github.com/spring-projects/spring-framework"
+
+
+class CollectReactPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_react_prs-issues"
+    repo_url = "https://github.com/facebook/react"
+
+
+class CollectAngularPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_angular_prs-issues"
+    repo_url = "https://github.com/angular/angular"
+
+
+class CollectWordpressPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_wordpress_prs-issues"
+    repo_url = "https://github.com/WordPress/WordPress"
+
+
+class CollectDockerMobyPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_docker_moby_prs-issues"
+    repo_url = "https://github.com/moby/moby"
+
+
+class CollectKubernetesPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_kubernetes_prs-issues"
     repo_url = "https://github.com/kubernetes/kubernetes"
 
 
-class CollectWiresharkPRSIssues(GitLabCollector):
-    pipeline_id = "collect-wireshark-prs-issues"
+class CollectXenProjectPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_xen_project_prs-issues"
+    repo_url = "https://github.com/xen-project/xen"
+
+
+class CollectVirtualboxPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_virtualbox_prs-issues"
+    repo_url = "https://github.com/mirror/vbox"
+
+
+class CollectContainerdPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_containerd_prs-issues"
+    repo_url = "https://github.com/containerd/containerd"
+
+
+class CollectAnsiblePRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_ansible_prs-issues"
+    repo_url = "https://github.com/ansible/ansible"
+
+
+class CollectTerraformPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_terraform_prs-issues"
+    repo_url = "https://github.com/hashicorp/terraform"
+
+
+class CollectTcpdumpPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_tcpdump_prs-issues"
+    repo_url = "https://github.com/the-tcpdump-group/tcpdump"
+
+
+class CollectJenkinsPRSIssuesPipeline(GitHubCollector):
+    pipeline_id = "collect_jenkins_prs-issues"
+    repo_url = "https://github.com/jenkinsci/jenkins"
+
+
+class CollectGitlabPRSIssuesPipeline(GitLabCollector):
+    pipeline_id = "collect_gitlab_prs-issues"
+    repo_url = "https://gitlab.com/gitlab-org/gitlab-foss"
+
+
+class CollectWiresharkPRSIssuesPipeline(GitLabCollector):
+    pipeline_id = "collect_wireshark_prs-issues"
     repo_url = "https://gitlab.com/wireshark/wireshark"
+
+
+class CollectQemuPRSIssuesPipeline(GitLabCollector):
+    pipeline_id = "collect_qemu_prs-issues"
+    repo_url = "https://gitlab.com/qemu-project/qemu"