Update affected_by_commits and fixed_by_commits to be separate fields in Advisory

Signed-off-by: ziad hany <ziadhany2016@gmail.com>

Author: ziadhany

vulnerabilities/importer.py

@@ -37,8 +37,8 @@
 from vulnerabilities.severity_systems import ScoringSystem
 from vulnerabilities.utils import classproperty
 from vulnerabilities.utils import get_reference_id
+from vulnerabilities.utils import is_commit
 from vulnerabilities.utils import is_cve
-from vulnerabilities.utils import nearest_patched_package
 from vulnerabilities.utils import purl_to_dict
 from vulnerabilities.utils import update_purl_version
 
@@ -194,6 +194,60 @@ def from_url(cls, url):
         return cls(url=url)
 
 
+@dataclasses.dataclass(eq=True)
+@functools.total_ordering
+class CodeCommitData:
+    commit_hash: str
+    vcs_url: str
+
+    commit_author: Optional[str] = None
+    commit_message: Optional[str] = None
+    commit_date: Optional[datetime.datetime] = None
+
+    def __post_init__(self):
+        if not self.commit_hash:
+            raise ValueError("Commit must have a non-empty commit_hash.")
+
+        if not is_commit(self.commit_hash):
+            raise ValueError("Commit must be a valid a commit_hash.")
+
+        if not self.vcs_url:
+            raise ValueError("Commit must have a non-empty vcs_url.")
+        if not isinstance(self.commit_hash, str):
+            self.commit_hash = str(self.commit_hash)
+
+    def __lt__(self, other):
+        if not isinstance(other, CodeCommitData):
+            return NotImplemented
+        return self._cmp_key() < other._cmp_key()
+
+    # TODO: Add cache
+    def _cmp_key(self):
+        return (self.commit_hash, self.vcs_url, self.commit_author, self.commit_message)
+
+    def to_dict(self) -> dict:
+        """Return a normalized dictionary representation of the commit."""
+        return {
+            "commit_hash": self.commit_hash,
+            "vcs_url": self.vcs_url,
+            "commit_author": self.commit_author,
+            "commit_message": self.commit_message,
+            "commit_date": self.commit_date,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        """Create a Commit instance from a dictionary."""
+        commit_date = data.get("commit_date")
+        return cls(
+            commit_hash=str(data.get("commit_hash", "")),
+            vcs_url=data.get("vcs_url", ""),
+            commit_author=data.get("commit_author"),
+            commit_message=data.get("commit_message"),
+            commit_date=datetime.datetime.fromisoformat(commit_date) if commit_date else None,
+        )
+
+
 class UnMergeablePackageError(Exception):
     """
     Raised when a package cannot be merged with another one.
@@ -444,6 +498,8 @@ class AdvisoryData:
     date_published: Optional[datetime.datetime] = None
     weaknesses: List[int] = dataclasses.field(default_factory=list)
     severities: List[VulnerabilitySeverity] = dataclasses.field(default_factory=list)
+    fixed_by_commits: List[CodeCommitData] = dataclasses.field(default_factory=list)
+    affected_by_commits: List[CodeCommitData] = dataclasses.field(default_factory=list)
     url: Optional[str] = None
     original_advisory_text: Optional[str] = None
 
@@ -476,6 +532,12 @@ def to_dict(self):
                 "severities": [sev.to_dict() for sev in self.severities],
                 "date_published": self.date_published.isoformat() if self.date_published else None,
                 "weaknesses": self.weaknesses,
+                "affected_by_commits": [
+                    affected_by_commit.to_dict() for affected_by_commit in self.affected_by_commits
+                ],
+                "fixed_by_commits": [
+                    fixed_by_commit.to_dict() for fixed_by_commit in self.fixed_by_commits
+                ],
                 "url": self.url if self.url else "",
             }
         return {
@@ -537,6 +599,19 @@ class AdvisoryDataV2:
     weaknesses: List[int] = dataclasses.field(default_factory=list)
     url: Optional[str] = None
 
+    # TODO
+    # Update from_dict and to_dict methods
+    # Update compute_checksum method
+    # Update BaseV2 Importer Pipeline
+    # Change related tests
+    # Add tests for these newly introduced fields
+    # have a strong test for insert_advisory_v2 method
+    # CodeCommitData importer
+    # remove commit_rank from CodeCommitData importer
+
+    fixed_by_commits: List[CodeCommitData] = dataclasses.field(default_factory=list)
+    affected_by_commits: List[CodeCommitData] = dataclasses.field(default_factory=list)
+
     def __post_init__(self):
         if self.date_published and not self.date_published.tzinfo:
             logger.warning(f"AdvisoryData with no tzinfo: {self!r}")
@@ -559,6 +634,12 @@ def to_dict(self):
             "references": [ref.to_dict() for ref in self.references],
             "date_published": self.date_published.isoformat() if self.date_published else None,
             "weaknesses": self.weaknesses,
+            "affected_by_commits": [
+                affected_by_commit.to_dict() for affected_by_commit in self.affected_by_commits
+            ],
+            "fixed_by_commits": [
+                fixed_by_commit.to_dict() for fixed_by_commit in self.fixed_by_commits
+            ],
             "url": self.url if self.url else "",
         }
 
@@ -578,6 +659,14 @@ def from_dict(cls, advisory_data):
             if date_published
             else None,
             "weaknesses": advisory_data["weaknesses"],
+            "affected_by_commits": [
+                CodeCommitData.from_dict(affected_by_commit)
+                for affected_by_commit in advisory_data["affected_by_commits"]
+            ],
+            "fixed_by_commits": [
+                CodeCommitData.from_dict(fixed_by_commit)
+                for fixed_by_commit in advisory_data["fixed_by_commits"]
+            ],
             "url": advisory_data.get("url") or None,
         }
         return cls(**transformed)

vulnerabilities/importers/curl.py

@@ -97,7 +97,7 @@ def parse_advisory_data(raw_data) -> AdvisoryData:
         ...     ]
         ... }
         >>> parse_advisory_data(raw_data)
-        AdvisoryData(advisory_id='', aliases=['CVE-2024-2379'], summary='QUIC certificate check bypass with wolfSSL', affected_packages=[AffectedPackage(package=PackageURL(type='generic', namespace='curl.se', name='curl', version=None, qualifiers={}, subpath=None), affected_version_range=GenericVersionRange(constraints=(VersionConstraint(comparator='=', version=SemverVersion(string='8.6.0')),)), fixed_version=SemverVersion(string='8.7.0'))], references=[Reference(reference_id='', reference_type='', url='https://curl.se/docs/CVE-2024-2379.html', severities=[VulnerabilitySeverity(system=Cvssv3ScoringSystem(identifier='cvssv3.1', name='CVSSv3.1 Base Score', url='https://www.first.org/cvss/v3-1/', notes='CVSSv3.1 base score and vector'), value='Low', scoring_elements='', published_at=None, url=None)]), Reference(reference_id='', reference_type='', url='https://hackerone.com/reports/2410774', severities=[])], references_v2=[], date_published=datetime.datetime(2024, 3, 27, 8, 0, tzinfo=datetime.timezone.utc), weaknesses=[297], severities=[], url='https://curl.se/docs/CVE-2024-2379.json', original_advisory_text=None)
+        AdvisoryData(advisory_id='', aliases=['CVE-2024-2379'], summary='QUIC certificate check bypass with wolfSSL', affected_packages=[AffectedPackage(package=PackageURL(type='generic', namespace='curl.se', name='curl', version=None, qualifiers={}, subpath=None), affected_version_range=GenericVersionRange(constraints=(VersionConstraint(comparator='=', version=SemverVersion(string='8.6.0')),)), fixed_version=SemverVersion(string='8.7.0'))], references=[Reference(reference_id='', reference_type='', url='https://curl.se/docs/CVE-2024-2379.html', severities=[VulnerabilitySeverity(system=Cvssv3ScoringSystem(identifier='cvssv3.1', name='CVSSv3.1 Base Score', url='https://www.first.org/cvss/v3-1/', notes='CVSSv3.1 base score and vector'), value='Low', scoring_elements='', published_at=None, url=None)]), Reference(reference_id='', reference_type='', url='https://hackerone.com/reports/2410774', severities=[])], references_v2=[], date_published=datetime.datetime(2024, 3, 27, 8, 0, tzinfo=datetime.timezone.utc), weaknesses=[297], severities=[], fixed_by_commits=[], affected_by_commits=[], url='https://curl.se/docs/CVE-2024-2379.json', original_advisory_text=None)
     """
 
     affected = get_item(raw_data, "affected")[0] if len(get_item(raw_data, "affected")) > 0 else []

vulnerabilities/pipes/advisory.py

@@ -17,6 +17,7 @@
 from typing import Union
 
 from django.db import transaction
+from django.db.models import Q
 from django.db.models.query import QuerySet
 
 from vulnerabilities.importer import AdvisoryData
@@ -29,6 +30,7 @@
 from vulnerabilities.models import AdvisoryWeakness
 from vulnerabilities.models import AffectedByPackageRelatedVulnerability
 from vulnerabilities.models import Alias
+from vulnerabilities.models import CodeCommit
 from vulnerabilities.models import FixingPackageRelatedVulnerability
 from vulnerabilities.models import Package
 from vulnerabilities.models import VulnerabilityReference
@@ -96,6 +98,42 @@ def get_or_create_advisory_weaknesses(weaknesses: List[str]) -> List[AdvisoryWea
     return list(AdvisoryWeakness.objects.filter(cwe_id__in=weaknesses))
 
 
+def get_or_create_advisory_code_commits(code_commits_data: List) -> List["CodeCommit"]:
+    """
+    Given a list of commit-like objects (each with commit_hash and vcs_url),
+    create any missing CodeCommit entries and return the full list of CodeCommit objects.
+    """
+    if not code_commits_data:
+        return []
+
+    pairs = [(c.commit_hash, c.vcs_url) for c in code_commits_data]
+
+    query = Q()
+    for commit_hash, vcs_url in pairs:
+        query |= Q(commit_hash=commit_hash, vcs_url=vcs_url)
+
+    existing_commits_qs = CodeCommit.objects.filter(query)
+    existing_pairs = set(existing_commits_qs.values_list("commit_hash", "vcs_url"))
+
+    to_create = [
+        CodeCommit(
+            commit_hash=c.commit_hash,
+            vcs_url=c.vcs_url,
+            commit_author=getattr(c, "commit_author", None),
+            commit_message=getattr(c, "commit_message", None),
+            commit_date=getattr(c, "commit_date", None),
+        )
+        for c in code_commits_data
+        if (c.commit_hash, c.vcs_url) not in existing_pairs
+    ]
+
+    if to_create:
+        CodeCommit.objects.bulk_create(to_create, ignore_conflicts=True)
+
+    all_commits = CodeCommit.objects.filter(query)
+    return list(all_commits)
+
+
 def insert_advisory(advisory: AdvisoryData, pipeline_id: str, logger: Callable = None):
     from vulnerabilities.utils import compute_content_id
 
@@ -150,6 +188,9 @@ def insert_advisory_v2(
     severities = get_or_create_advisory_severities(severities=advisory.severities)
     weaknesses = get_or_create_advisory_weaknesses(weaknesses=advisory.weaknesses)
     content_id = compute_content_id(advisory_data=advisory)
+    affected_by_commits = get_or_create_advisory_code_commits(advisory.affected_by_commits)
+    fixed_by_commits = get_or_create_advisory_code_commits(advisory.fixed_by_commits)
+
     try:
         default_data = {
             "datasource_id": pipeline_id,
@@ -216,6 +257,9 @@ def insert_advisory_v2(
             impact.affecting_packages.add(*affected_packages_v2)
             impact.fixed_by_packages.add(*fixed_packages_v2)
 
+            impact.affecting_commits.add(*affected_by_commits)
+            impact.fixed_by_commits.add(*fixed_by_commits)
+
     return advisory_obj
 
 

vulnerabilities/tests/pipes/test_advisory.py

@@ -19,12 +19,15 @@
 from vulnerabilities import models
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import AffectedPackage
+from vulnerabilities.importer import CodeCommitData
 from vulnerabilities.importer import Reference
 from vulnerabilities.models import AdvisoryAlias
 from vulnerabilities.models import AdvisoryReference
 from vulnerabilities.models import AdvisorySeverity
 from vulnerabilities.models import AdvisoryWeakness
+from vulnerabilities.models import CodeCommit
 from vulnerabilities.pipes.advisory import get_or_create_advisory_aliases
+from vulnerabilities.pipes.advisory import get_or_create_advisory_code_commits
 from vulnerabilities.pipes.advisory import get_or_create_advisory_references
 from vulnerabilities.pipes.advisory import get_or_create_advisory_severities
 from vulnerabilities.pipes.advisory import get_or_create_advisory_weaknesses
@@ -44,6 +47,18 @@ def setUp(self):
                 )
             ],
             references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")],
+            affected_by_commits=[
+                CodeCommitData(
+                    commit_hash="9ff29db8ec3adefefce0d37c3c9b5b2c22e59fac",
+                    vcs_url="https://github.com/aboutcode-org/vulnerablecode",
+                )
+            ],
+            fixed_by_commits=[
+                CodeCommitData(
+                    commit_hash="9ff29db8ec3adefefce0d37c3c9b5b2c22e59fac",
+                    vcs_url="https://github.com/aboutcode-org/vulnerablecode",
+                )
+            ],
             date_published=timezone.now(),
             url="https://test.com",
         )
@@ -160,6 +175,23 @@ def advisory_references():
     ]
 
 
+@pytest.fixture
+def advisory_commit():
+    return [
+        CodeCommitData(
+            commit_hash="ef1659c01708b2111d6f06e2aa32f0f9d8768e10",
+            vcs_url="https://github.com/aboutcode-org/vulnerablecode",
+            commit_author="tester1",
+            commit_message="message1",
+            commit_date=datetime.now(),
+        ),
+        CodeCommitData(
+            commit_hash="eccbb45ac2d9c0eb7e22ea82d1fc49f9f4cda818",
+            vcs_url="https://github.com/aboutcode-org/vulnerablecode",
+        ),
+    ]
+
+
 @pytest.fixture
 def advisory_severities():
     class Severity:
@@ -225,3 +257,13 @@ def test_get_or_create_advisory_weaknesses(advisory_weaknesses):
     for w in weaknesses:
         assert isinstance(w, AdvisoryWeakness)
         assert w.cwe_id in advisory_weaknesses
+
+
+@pytest.mark.django_db
+def test_get_or_create_advisory_commit(advisory_commit):
+    commits = get_or_create_advisory_code_commits(advisory_commit)
+    assert len(commits) == len(advisory_commit)
+    for commit in commits:
+        assert isinstance(commit, CodeCommit)
+        assert commit.commit_hash in [c.commit_hash for c in advisory_commit]
+        assert commit.vcs_url in [c.vcs_url for c in advisory_commit]

vulnerabilities/tests/pipes/test_vulnerablecode_importer_pipeline_v2.py

@@ -18,7 +18,9 @@
 
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import AffectedPackageV2
+from vulnerabilities.importer import CodeCommitData
 from vulnerabilities.models import AdvisoryV2
+from vulnerabilities.models import CodeCommit
 from vulnerabilities.models import ImpactedPackage
 from vulnerabilities.models import PackageV2
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
@@ -59,6 +61,18 @@ def dummy_advisory():
             ),
         ],
         advisory_id="ADV-123",
+        fixed_by_commits=[
+            CodeCommitData(
+                commit_hash="9ff29db8ec3adefefce0d37c3c9b5b2c22e59fac",
+                vcs_url="https://github.com/aboutcode-org/vulnerablecode",
+            )
+        ],
+        affected_by_commits=[
+            CodeCommitData(
+                commit_hash="ab99939678dc36b3bee0f366493df1aeef521df4",
+                vcs_url="https://github.com/aboutcode-org/vulnerablecode",
+            )
+        ],
         date_published=datetime.now() - timedelta(days=10),
         url="https://example.com/advisory/1",
     )
@@ -92,4 +106,5 @@ def test_advisory_import_atomicity(dummy_importer):
     dummy_importer.collect_and_store_advisories()
     assert AdvisoryV2.objects.count() == 1
     assert ImpactedPackage.objects.count() == 2
+    assert CodeCommit.objects.count() == 2
     assert PackageV2.objects.count() == 4

vulnerabilities/tests/test_data/archlinux/archlinux_advisoryv2-expected.json

@@ -29,6 +29,8 @@
     "severities": [],
     "date_published": null,
     "weaknesses": [],
+    "affected_by_commits": [],
+    "fixed_by_commits": [],
     "url": "https://security.archlinux.org/AVG-2781.json"
   },
   {
@@ -63,6 +65,8 @@
     "severities": [],
     "date_published": null,
     "weaknesses": [],
+    "affected_by_commits": [],
+    "fixed_by_commits": [],
     "url": "https://security.archlinux.org/AVG-2780.json"
   },
   {
@@ -101,6 +105,8 @@
     "severities": [],
     "date_published": null,
     "weaknesses": [],
+    "affected_by_commits": [],
+    "fixed_by_commits": [],
     "url": "https://security.archlinux.org/AVG-4.json"
   }
 ]