Add support for Reference Fix Commits improver

Update the pipeline and fix the test

Signed-off-by: ziad hany <ziadhany2016@gmail.com>

Author: ziadhany

vulnerabilities/importers/__init__.py

@@ -82,6 +82,7 @@
 from vulnerabilities.pipelines.v2_importers import ubuntu_osv_importer as ubuntu_osv_importer_v2
 from vulnerabilities.pipelines.v2_importers import vulnrichment_importer as vulnrichment_importer_v2
 from vulnerabilities.pipelines.v2_importers import xen_importer as xen_importer_v2
+from vulnerabilities.pipelines.v2_improvers import reference_collect_commits
 from vulnerabilities.utils import create_registry
 
 IMPORTERS_REGISTRY = create_registry(
@@ -127,6 +128,7 @@
         nginx_importer.NginxImporterPipeline,
         pysec_importer.PyPIImporterPipeline,
         fireeye_importer_v2.FireeyeImporterPipeline,
+        reference_collect_commits.CollectReferencesFixCommitsPipeline,
         apache_tomcat.ApacheTomcatImporter,
         postgresql.PostgreSQLImporter,
         debian.DebianImporter,

vulnerabilities/pipelines/v2_improvers/reference_collect_commits.py

@@ -0,0 +1,76 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from aboutcode.pipeline import LoopProgress
+from packageurl.contrib.purl2url import purl2url
+from packageurl.contrib.url2purl import url2purl
+
+from aboutcode.federated import get_core_purl
+from vulnerabilities.models import AdvisoryV2
+from vulnerabilities.models import PackageCommitPatch
+from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+from vulnerabilities.pipes.advisory import VCS_URLS_SUPPORTED_TYPES
+from vulnerabilities.tests.test_export import package
+from vulnerabilities.utils import is_commit
+
+
+class CollectReferencesFixCommitsPipeline(VulnerableCodeBaseImporterPipelineV2):
+    """
+    Improver pipeline to scout References/Patch and create PackageCommitPatch entries.
+    """
+
+    pipeline_id = "collect_ref_fix_commits_v2"
+    license_expression = None
+
+    @classmethod
+    def steps(cls):
+        return (cls.collect_and_store_fix_commits,)
+
+    def get_vcs_commit(self, url):
+        """Extracts and VCS URL and commit hash from URL.
+        >> get_vcs_commit('https://github.com/aboutcode-org/vulnerablecode/commit/98e516011d6e096e25247b82fc5f196bbeecff10')
+        ('https://github.com/aboutcode-org/vulnerablecode', '98e516011d6e096e25247b82fc5f196bbeecff10')
+        >> get_vcs_commit('https://github.com/aboutcode-org/vulnerablecode/pull/1974')
+        None
+        """
+        purl = url2purl(url)
+        if not purl or purl.type not in VCS_URLS_SUPPORTED_TYPES:
+            return None
+
+        version = getattr(purl, "version", None)
+        if not version or not is_commit(version):
+            return None
+
+        vcs_url = purl2url(get_core_purl(purl).to_string())
+        return (vcs_url, version) if vcs_url else None
+
+    def collect_and_store_fix_commits(self):
+        impacted_packages_advisories = (
+            AdvisoryV2.objects.filter(impacted_packages__isnull=False)
+            .prefetch_related("references", "patches", "impacted_packages")
+            .distinct()
+        )
+
+        progress = LoopProgress(
+            total_iterations=impacted_packages_advisories.count(), logger=self.log
+        )
+        for adv in progress.iter(impacted_packages_advisories.paginated(per_page=500)):
+            urls = {r.url for r in adv.references.all()} | {p.patch_url for p in adv.patches.all()}
+            impacted_packages = list(adv.impacted_packages.all())
+
+            for url in urls:
+                vcs_data = self.get_vcs_commit(url)
+                if not vcs_data:
+                    continue
+
+                vcs_url, commit_hash = vcs_data
+                package_commit_obj, _ = PackageCommitPatch.objects.get_or_create(
+                    vcs_url=vcs_url, commit_hash=commit_hash
+                )
+                package_commit_obj.fixed_impacted_packages.add(*impacted_packages)

vulnerabilities/tests/pipelines/v2_improvers/test_ref_collect_commits_v2.py

@@ -0,0 +1,105 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+
+from datetime import datetime
+
+import pytest
+
+from vulnerabilities.models import AdvisoryReference
+from vulnerabilities.models import AdvisoryV2
+from vulnerabilities.models import ImpactedPackage
+from vulnerabilities.models import PackageCommitPatch
+from vulnerabilities.models import PackageV2
+from vulnerabilities.pipelines.v2_improvers.reference_collect_commits import (
+    CollectReferencesFixCommitsPipeline,
+)
+
+
+@pytest.mark.django_db
+def test_is_vcs_url_already_processed_true():
+    advisory = AdvisoryV2.objects.create(
+        advisory_id="CVE-2025-9999",
+        datasource_id="test-ds",
+        avid="test-ds/CVE-2025-9999",
+        url="https://example.com/advisory/CVE-2025-9999",
+        unique_content_id="11111",
+        date_collected=datetime.now(),
+    )
+    package = PackageV2.objects.create(
+        type="bar",
+        name="foo",
+        version="1.0",
+    )
+    impact = ImpactedPackage.objects.create(advisory=advisory)
+    impact.affecting_packages.add(package)
+    package_commit_patch = PackageCommitPatch.objects.create(
+        vcs_url="https://github.com/user/repo/commit/6bd301819f8f69331a55ae2336c8b111fc933f3d",
+        commit_hash="6bd301819f8f69331a55ae2336c8b111fc933f3d",
+    )
+    impact.fixed_by_package_commit_patches.add(package_commit_patch)
+
+
+@pytest.mark.django_db
+def test_collect_fix_commits_pipeline_creates_entry():
+    advisory = AdvisoryV2.objects.create(
+        advisory_id="CVE-2025-1000",
+        datasource_id="test-ds",
+        avid="test-ds/CVE-2025-1000",
+        url="https://example.com/advisory/CVE-2025-1000",
+        unique_content_id="11111",
+        date_collected=datetime.now(),
+    )
+    package = PackageV2.objects.create(
+        type="foo",
+        name="testpkg",
+        version="1.0",
+    )
+    reference = AdvisoryReference.objects.create(
+        url="https://github.com/test/testpkg/commit/6bd301819f8f69331a55ae2336c8b111fc933f3d"
+    )
+    impact = ImpactedPackage.objects.create(advisory=advisory)
+    impact.affecting_packages.add(package)
+    advisory.references.add(reference)
+
+    pipeline = CollectReferencesFixCommitsPipeline()
+    pipeline.collect_and_store_fix_commits()
+
+    package_commit_patch = PackageCommitPatch.objects.all()
+
+    assert package_commit_patch.count() == 1
+    fix = package_commit_patch.first()
+    assert fix.commit_hash == "6bd301819f8f69331a55ae2336c8b111fc933f3d"
+    assert fix.vcs_url == "https://github.com/test/testpkg"
+
+
+@pytest.mark.django_db
+def test_collect_fix_commits_pipeline_skips_non_commit_urls():
+    advisory = AdvisoryV2.objects.create(
+        advisory_id="CVE-2025-2000",
+        datasource_id="test-ds",
+        avid="test-ds/CVE-2025-2000",
+        url="https://example.com/advisory/CVE-2025-2000",
+        unique_content_id="11111",
+        date_collected=datetime.now(),
+    )
+    package = PackageV2.objects.create(
+        type="pypi",
+        name="otherpkg",
+        version="2.0",
+    )
+    impact = ImpactedPackage.objects.create(advisory=advisory)
+    impact.affecting_packages.add(package)
+
+    reference = AdvisoryReference.objects.create(
+        url="https://github.com/test/testpkg/issues/12"
+    )  # invalid reference 1
+    advisory.references.add(reference)
+
+    pipeline = CollectReferencesFixCommitsPipeline()
+    pipeline.collect_and_store_fix_commits()
+    assert PackageCommitPatch.objects.count() == 0