Move the CollectVCSFixCommitPipeline base pipelines to pipes

Remove CollectOpensslFixCommits Pipeline

Signed-off-by: ziad hany <ziadhany2016@gmail.com>

Author: ziadhany

vulnerabilities/importers/__init__.py

@@ -153,7 +153,6 @@
         collect_fix_commits_v2.CollectRustFixCommitsPipeline,
         collect_fix_commits_v2.CollectOpenjdkFixCommitsPipeline,
         collect_fix_commits_v2.CollectSwiftFixCommitsPipeline,
-        collect_fix_commits_v2.CollectOpensslFixCommitsPipeline,
         collect_fix_commits_v2.CollectDjangoFixCommitsPipeline,
         collect_fix_commits_v2.CollectRailsFixCommitsPipeline,
         collect_fix_commits_v2.CollectLaravelFixCommitsPipeline,

vulnerabilities/pipelines/__init__.py

@@ -8,11 +8,7 @@
 #
 
 import logging
-import re
-import shutil
-import tempfile
 import traceback
-from collections import defaultdict
 from datetime import datetime
 from datetime import timezone
 from timeit import default_timer as timer
@@ -23,12 +19,8 @@
 from aboutcode.pipeline import LoopProgress
 from aboutcode.pipeline import PipelineDefinition
 from aboutcode.pipeline import humanize_time
-from git import Repo
-from packageurl.contrib.url2purl import url2purl
 
 from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importer import AffectedPackageV2
-from vulnerabilities.importer import PackageCommitPatchData
 from vulnerabilities.improver import MAX_CONFIDENCE
 from vulnerabilities.models import Advisory
 from vulnerabilities.models import PipelineRun
@@ -329,109 +321,3 @@ def collect_and_store_advisories(self):
                 continue
 
         self.log(f"Successfully collected {collected_advisory_count:,d} advisories")
-
-
-class CollectVCSFixCommitPipeline(VulnerableCodeBaseImporterPipelineV2):
-    """
-    Pipeline to collect fix commits from any git repository.
-    """
-
-    repo_url: str
-    patterns: list[str] = [
-        r"\bCVE-\d{4}-\d{4,19}\b",
-        r"GHSA-[2-9cfghjmpqrvwx]{4}-[2-9cfghjmpqrvwx]{4}-[2-9cfghjmpqrvwx]{4}",
-    ]
-
-    @classmethod
-    def steps(cls):
-        return (
-            cls.clone,
-            cls.collect_and_store_advisories,
-            cls.clean_downloads,
-        )
-
-    def clone(self):
-        """Clone the repository."""
-        self.repo = Repo.clone_from(
-            url=self.repo_url,
-            to_path=tempfile.mkdtemp(),
-            bare=True,
-            no_checkout=True,
-            multi_options=["--filter=blob:none"],
-        )
-
-    def advisories_count(self) -> int:
-        return 0
-
-    def extract_vulnerability_id(self, commit) -> list[str]:
-        """
-        Extract vulnerability id from a commit message.
-        Returns a list of matched vulnerability IDs
-        """
-        matches = []
-        for pattern in self.patterns:
-            found = re.findall(pattern, commit.message, flags=re.IGNORECASE)
-            matches.extend(found)
-        return matches
-
-    def collect_fix_commits(self):
-        """
-        Iterate through repository commits and group them by vulnerability identifiers.
-        return a list with (vuln_id, [(commit_id, commit_message)]).
-        """
-        self.log("Processing git repository fix commits (grouped by vulnerability IDs).")
-
-        grouped_commits = defaultdict(list)
-        for commit in self.repo.iter_commits("--all"):
-            matched_ids = self.extract_vulnerability_id(commit)
-            if not matched_ids:
-                continue
-
-            commit_id = commit.hexsha
-            commit_message = commit.message.strip()
-
-            for vuln_id in matched_ids:
-                grouped_commits[vuln_id].append((commit_id, commit_message))
-
-        self.log(f"Found {len(grouped_commits)} vulnerabilities with related commits.")
-        self.log("Finished processing all commits.")
-        return grouped_commits
-
-    def collect_advisories(self):
-        """
-        Generate AdvisoryData objects for each vulnerability ID grouped with its related commits.
-        """
-        self.log("Generating AdvisoryData objects from grouped commits.")
-        grouped_commits = self.collect_fix_commits()
-        purl = url2purl(self.repo_url)
-
-        for vuln_id, commits_data in grouped_commits.items():
-            if not commits_data or not vuln_id:
-                continue
-
-            commit_hash_set = {commit_hash for commit_hash, _ in commits_data}
-            affected_packages = [
-                AffectedPackageV2(
-                    package=purl,
-                    fixed_by_commit_patches=[
-                        PackageCommitPatchData(vcs_url=self.repo_url, commit_hash=commit_hash)
-                        for commit_hash in commit_hash_set
-                    ],
-                )
-            ]
-
-            yield AdvisoryData(
-                advisory_id=vuln_id,
-                affected_packages=affected_packages,
-                url=self.repo_url,
-            )
-
-    def clean_downloads(self):
-        """Cleanup any temporary repository data."""
-        self.log("Cleaning up local repository resources.")
-        if hasattr(self, "repo") and self.repo.working_dir:
-            shutil.rmtree(path=self.repo.working_dir)
-
-    def on_failure(self):
-        """Ensure cleanup is always performed on failure."""
-        self.clean_downloads()

vulnerabilities/pipelines/v2_importers/collect_fix_commits.py

@@ -1,191 +1,186 @@
-from vulnerabilities.pipelines import CollectVCSFixCommitPipeline
+from vulnerabilities.pipes.vcs_collector_utils import CollectVCSFixCommitPipeline
 
 
 class CollectLinuxFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_linux_fix_commits"
-    repo_url = "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git"
+    repo_url = "https://github.com/torvalds/linux"
 
 
 class CollectBusyBoxFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_busybox_fix_commits"
-    repo_url = "https://github.com/mirror/busybox.git"
+    repo_url = "https://github.com/mirror/busybox"
 
 
 class CollectNginxFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_nginx_fix_commits"
-    repo_url = "https://github.com/nginx/nginx.git"
+    repo_url = "https://github.com/nginx/nginx"
 
 
 class CollectApacheTomcatFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_apache_tomcat_fix_commits"
-    repo_url = "https://github.com/apache/tomcat.git"
+    repo_url = "https://github.com/apache/tomcat"
 
 
 class CollectMysqlServerFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_mysql_server_fix_commits"
-    repo_url = "https://github.com/mysql/mysql-server.git"
+    repo_url = "https://github.com/mysql/mysql-server"
 
 
 class CollectPostgresqlFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_postgresql_fix_commits"
-    repo_url = "https://github.com/postgres/postgres.git"
+    repo_url = "https://github.com/postgres/postgres"
 
 
 class CollectMongodbFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_mongodb_fix_commits"
-    repo_url = "https://github.com/mongodb/mongo.git"
+    repo_url = "https://github.com/mongodb/mongo"
 
 
 class CollectRedisFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_redis_fix_commits"
-    repo_url = "https://github.com/redis/redis.git"
+    repo_url = "https://github.com/redis/redis"
 
 
 class CollectSqliteFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_sqlite_fix_commits"
-    repo_url = "https://github.com/sqlite/sqlite.git"
+    repo_url = "https://github.com/sqlite/sqlite"
 
 
 class CollectPhpFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_php_fix_commits"
-    repo_url = "https://github.com/php/php-src.git"
+    repo_url = "https://github.com/php/php-src"
 
 
 class CollectPythonCpythonFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_python_cpython_fix_commits"
-    repo_url = "https://github.com/python/cpython.git"
+    repo_url = "https://github.com/python/cpython"
 
 
 class CollectRubyFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_ruby_fix_commits"
-    repo_url = "https://github.com/ruby/ruby.git"
+    repo_url = "https://github.com/ruby/ruby"
 
 
 class CollectGoFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_go_fix_commits"
-    repo_url = "https://github.com/golang/go.git"
+    repo_url = "https://github.com/golang/go"
 
 
 class CollectNodeJsFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_node_js_fix_commits"
-    repo_url = "https://github.com/nodejs/node.git"
+    repo_url = "https://github.com/nodejs/node"
 
 
 class CollectRustFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_rust_fix_commits"
-    repo_url = "https://github.com/rust-lang/rust.git"
+    repo_url = "https://github.com/rust-lang/rust"
 
 
 class CollectOpenjdkFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_openjdk_fix_commits"
-    repo_url = "https://github.com/openjdk/jdk.git"
+    repo_url = "https://github.com/openjdk/jdk"
 
 
 class CollectSwiftFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_swift_fix_commits"
-    repo_url = "https://github.com/swiftlang/swift.git"
-
-
-class CollectOpensslFixCommitsPipeline(CollectVCSFixCommitPipeline):
-    pipeline_id = "collect_openssl_fix_commits"
-    repo_url = "https://github.com/openssl/openssl.git"
+    repo_url = "https://github.com/swiftlang/swift"
 
 
 class CollectDjangoFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_django_fix_commits"
-    repo_url = "https://github.com/django/django.git"
+    repo_url = "https://github.com/django/django"
 
 
 class CollectRailsFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_rails_fix_commits"
-    repo_url = "https://github.com/rails/rails.git"
+    repo_url = "https://github.com/rails/rails"
 
 
 class CollectLaravelFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_laravel_fix_commits"
-    repo_url = "https://github.com/laravel/framework.git"
+    repo_url = "https://github.com/laravel/framework"
 
 
 class CollectSpringFrameworkFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_spring_framework_fix_commits"
-    repo_url = "https://github.com/spring-projects/spring-framework.git"
+    repo_url = "https://github.com/spring-projects/spring-framework"
 
 
 class CollectReactFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_react_fix_commits"
-    repo_url = "https://github.com/facebook/react.git"
+    repo_url = "https://github.com/facebook/react"
 
 
 class CollectAngularFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_angular_fix_commits"
-    repo_url = "https://github.com/angular/angular.git"
+    repo_url = "https://github.com/angular/angular"
 
 
 class CollectWordpressFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_wordpress_fix_commits"
-    repo_url = "https://github.com/WordPress/WordPress.git"
+    repo_url = "https://github.com/WordPress/WordPress"
 
 
 class CollectDockerMobyFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_docker_moby_fix_commits"
-    repo_url = "https://github.com/moby/moby.git"
+    repo_url = "https://github.com/moby/moby"
 
 
 class CollectKubernetesFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_kubernetes_fix_commits"
-    repo_url = "https://github.com/kubernetes/kubernetes.git"
+    repo_url = "https://github.com/kubernetes/kubernetes"
 
 
 class CollectQemuFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_qemu_fix_commits"
-    repo_url = "https://gitlab.com/qemu-project/qemu.git"
+    repo_url = "https://gitlab.com/qemu-project/qemu"
 
 
 class CollectXenProjectFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_xen_project_fix_commits"
-    repo_url = "https://github.com/xen-project/xen.git"
+    repo_url = "https://github.com/xen-project/xen"
 
 
 class CollectVirtualboxFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_virtualbox_fix_commits"
-    repo_url = "https://github.com/mirror/vbox.git"
+    repo_url = "https://github.com/mirror/vbox"
 
 
 class CollectContainerdFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_containerd_fix_commits"
-    repo_url = "https://github.com/containerd/containerd.git"
+    repo_url = "https://github.com/containerd/containerd"
 
 
 class CollectAnsibleFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_ansible_fix_commits"
-    repo_url = "https://github.com/ansible/ansible.git"
+    repo_url = "https://github.com/ansible/ansible"
 
 
 class CollectTerraformFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_terraform_fix_commits"
-    repo_url = "https://github.com/hashicorp/terraform.git"
+    repo_url = "https://github.com/hashicorp/terraform"
 
 
 class CollectWiresharkFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_wireshark_fix_commits"
-    repo_url = "https://gitlab.com/wireshark/wireshark.git"
+    repo_url = "https://gitlab.com/wireshark/wireshark"
 
 
 class CollectTcpdumpFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_tcpdump_fix_commits"
-    repo_url = "https://github.com/the-tcpdump-group/tcpdump.git"
+    repo_url = "https://github.com/the-tcpdump-group/tcpdump"
 
 
 class CollectGitFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_git_fix_commits"
-    repo_url = "https://github.com/git/git.git"
+    repo_url = "https://github.com/git/git"
 
 
 class CollectJenkinsFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_jenkins_fix_commits"
-    repo_url = "https://github.com/jenkinsci/jenkins.git"
+    repo_url = "https://github.com/jenkinsci/jenkins"
 
 
 class CollectGitlabFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_gitlab_fix_commits"
-    repo_url = "https://gitlab.com/gitlab-org/gitlab-foss.git"
+    repo_url = "https://gitlab.com/gitlab-org/gitlab-foss"