Add support for Reference Fix Commits improver

Signed-off-by: ziad hany <ziadhany2016@gmail.com>

Author: ziadhany

vulnerabilities/importers/__init__.py

@@ -77,6 +77,7 @@
 from vulnerabilities.pipelines.v2_importers import ubuntu_osv_importer as ubuntu_osv_importer_v2
 from vulnerabilities.pipelines.v2_importers import vulnrichment_importer as vulnrichment_importer_v2
 from vulnerabilities.pipelines.v2_importers import xen_importer as xen_importer_v2
+from vulnerabilities.pipelines.v2_improvers import reference_collect_commits
 from vulnerabilities.utils import create_registry
 
 IMPORTERS_REGISTRY = create_registry(
@@ -118,6 +119,7 @@
         nginx_importer.NginxImporterPipeline,
         pysec_importer.PyPIImporterPipeline,
         fireeye_importer_v2.FireeyeImporterPipeline,
+        reference_collect_commits.CollectReferencesFixCommitsPipeline,
         apache_tomcat.ApacheTomcatImporter,
         postgresql.PostgreSQLImporter,
         debian.DebianImporter,

vulnerabilities/pipelines/v2_improvers/collect_commits.py

@@ -0,0 +1,71 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from aboutcode.pipeline import LoopProgress
+from packageurl.contrib.purl2url import purl2url
+from packageurl.contrib.url2purl import url2purl
+
+from aboutcode.federated import get_core_purl
+from vulnerabilities.models import AdvisoryV2
+from vulnerabilities.models import PackageCommitPatch
+from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+from vulnerabilities.pipes.advisory import VCS_URLS_SUPPORTED_TYPES
+from vulnerabilities.utils import is_commit
+
+
+class CollectReferencesFixCommitsPipeline(VulnerableCodeBaseImporterPipelineV2):
+    """
+    Improver pipeline to scout References and create PackageCommitPatch entries.
+    """
+
+    pipeline_id = "collect_fix_commits_v2"
+    license_expression = None
+
+    @classmethod
+    def steps(cls):
+        return (cls.collect_and_store_fix_commits,)
+
+    def collect_and_store_fix_commits(self):
+        affected_advisories = (
+            AdvisoryV2.objects.filter(impacted_packages__affecting_packages__isnull=False)
+            .prefetch_related("impacted_packages__affecting_packages", "references")
+            .distinct()
+        )
+
+        self.log(f"Processing {affected_advisories.count():,d} references to collect fix commits.")
+
+        package_commit_patch_count = 0
+        progress = LoopProgress(total_iterations=affected_advisories.count(), logger=self.log)
+        for adv in progress.iter(affected_advisories.paginated(per_page=500)):
+            for reference in adv.references.all():
+                purl = url2purl(reference.url)
+                if not purl or (purl.type not in VCS_URLS_SUPPORTED_TYPES) or not is_commit(purl.version):
+                    continue
+
+                base_purl = get_core_purl(purl)
+                base_purl_str = base_purl.to_string()
+                vcs_url = purl2url(base_purl_str)
+                commit_hash = purl.version
+
+                if not vcs_url:
+                    continue
+
+                package_commit_patch, created = PackageCommitPatch.objects.get_or_create(
+                    vcs_url=vcs_url,
+                    commit_hash=commit_hash,
+                )
+
+                if created:
+                    for impact in adv.impacted_packages.all():
+                        impact.fixed_by_package_commit_patches.add(package_commit_patch)
+                    package_commit_patch_count += 1
+
+        self.log(
+            f"Successfully created {package_commit_patch_count:,d} PackageCommitPatch entries."
+        )