Merge branch 'main' into feat/cloudvulndb-importer

Author: Tednoob17

CHANGELOG.rst

@@ -13,10 +13,16 @@ Version v37.0.0
 - We have added new models AdvisoryV2, AdvisoryAlias, AdvisoryReference, AdvisorySeverity, AdvisoryWeakness, PackageV2 and CodeFixV2.
 - We are using ``avid`` as an internal advisory ID for uniquely identifying advisories.
 - We have a new route ``/v2`` which only support package search which has information on packages that are reported to be affected or fixing by advisories.
-- This version introduces ``/api/v2/advisories-packages`` which has information on packages that are reported to be affected or fixing by advisories.
+- This version introduces ``/api/v3/packages`` which has information on packages that are reported to be affected or fixing by advisories.
 - Pipeline Dashboard improvements #1920.
 - Throttle API requests based on user permissions #1909.
 - Add pipeline to compute Advisory ToDos #1764
+- Use related advisory severity to calculate exploitibility, weighted severity and risk scores
+- Migrate all importers to use the new advisory models. All new advisories have a unique AVID and all importers will use this AVID as the unique identifier for advisories instead of CVE ID or other identifiers used by the data sources #1881.
+- Handle advisories with same and related data https://github.com/aboutcode-org/vulnerablecode/issues/2099.
+- Add a pipeline for exporting VulnerableCode data to FederatedCode #2110.
+- Plan storing of exploits and EPSS based advisories #2069.
+
 
 Version v36.1.3
 ---------------------

PIPELINES-AVID.rst

@@ -0,0 +1,74 @@
+.. list-table:: Pipeline AVID Mapping
+   :header-rows: 1
+   :widths: 35 65
+
+   * - pipeline name
+     - AVID
+   * - alpine_linux_importer_v2
+     - {package_name}/{distroversion}/{version}/{vulnerability_id}
+   * - aosp_dataset_fix_commits
+     - CVE ID of the record
+   * - apache_httpd_importer_v2
+     - CVE ID of the record
+   * - apache_kafka_importer_v2
+     - CVE ID of the record
+   * - apache_tomcat_importer_v2
+     - {page_id}/{cve_id}
+   * - archlinux_importer_v2
+     - AVG ID of the record
+   * - curl_importer_v2
+     - CURL-CVE ID of the record
+   * - debian_importer_v2
+     - {package_name}/{debian_record_id}
+   * - elixir_security_importer_v2
+     - {package_name}/{file_id}
+   * - epss_importer_v2
+     - CVE ID of the record
+   * - fireeye_importer_v2
+     - {file_id}
+   * - gentoo_importer_v2
+     - GLSA ID of the record
+   * - github_osv_importer_v2
+     - ID of the OSV record
+   * - gitlab_importer_v2
+     - Identifier of the GitLab community advisory record
+   * - istio_importer_v2
+     - ISTIO-SECURITY-<ID>
+   * - mattermost_importer_v2
+     - MMSA-<ID>
+   * - mozilla_importer_v2
+     - MFSA-<ID>
+   * - nginx_importer_v2
+     - First alias of the record
+   * - nodejs_security_wg
+     - NPM-<ID>
+   * - nvd_importer_v2
+     - CVE ID of the record
+   * - openssl_importer_v2
+     - CVE ID of the record
+   * - oss_fuzz_importer_v2
+     - ID of the OSV record
+   * - postgresql_importer_v2
+     - CVE ID of the record
+   * - project-kb-msr-2019_v2
+     - Vulnerability ID of the record
+   * - project-kb-statements_v2
+     - Vulnerability ID of the record
+   * - pypa_importer_v2
+     - ID of the OSV record
+   * - pysec_importer_v2
+     - ID of the OSV record
+   * - redhat_importer_v2
+     - RHSA ID of the record
+   * - retiredotnet_importer_v2
+     - retiredotnet-{file_id}
+   * - ruby_importer_v2
+     - {file_id}
+   * - suse_importer_v2
+     - CVE ID of the record
+   * - ubuntu_osv_importer_v2
+     - ID of the OSV record
+   * - vulnrichment_importer_v2
+     - CVE ID of the record
+   * - xen_importer_v2
+     - XSA-<ID>

vulnerabilities/improvers/__init__.py

@@ -20,6 +20,7 @@
 from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline
 from vulnerabilities.pipelines import remove_duplicate_advisories
 from vulnerabilities.pipelines.v2_improvers import collect_ssvc_trees
+from vulnerabilities.pipelines.v2_improvers import compute_advisory_content_hash
 from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2
 from vulnerabilities.pipelines.v2_improvers import compute_package_risk as compute_package_risk_v2
 from vulnerabilities.pipelines.v2_improvers import (
@@ -74,5 +75,6 @@
         compute_advisory_todo.ComputeToDo,
         collect_ssvc_trees.CollectSSVCPipeline,
         relate_severities.RelateSeveritiesPipeline,
+        compute_advisory_content_hash.ComputeAdvisoryContentHash,
     ]
 )

vulnerabilities/migrations/0116_advisoryv2_advisory_content_hash.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.2.11 on 2026-03-11 08:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0115_impactedpackageaffecting_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="advisoryv2",
+            name="advisory_content_hash",
+            field=models.CharField(
+                blank=True,
+                help_text="A unique hash computed from the content of the advisory used to identify advisories with the same content.",
+                max_length=64,
+                null=True,
+            ),
+        ),
+    ]

vulnerabilities/models.py

@@ -3010,6 +3010,13 @@ class AdvisoryV2(models.Model):
         help_text="Related advisories that are used to calculate the severity of this advisory.",
     )
 
+    advisory_content_hash = models.CharField(
+        max_length=64,
+        blank=True,
+        null=True,
+        help_text="A unique hash computed from the content of the advisory used to identify advisories with the same content.",
+    )
+
     @property
     def risk_score(self):
         """
@@ -3078,35 +3085,6 @@ def get_aliases(self):
         """
         return self.aliases.all()
 
-    def compute_advisory_content(self):
-        """
-        Compute a unique content hash for an advisory by normalizing its data and hashing it.
-
-        :param advisory: An Advisory object
-        :return: SHA-256 hash digest as content hash
-        """
-        normalized_data = {
-            "summary": normalize_text(self.summary),
-            "impacted_packages": sorted(
-                [impact.to_dict() for impact in self.impacted_packages.all()],
-                key=lambda x: json.dumps(x, sort_keys=True),
-            ),
-            "patches": sorted(
-                [patch.to_patch_data().to_dict() for patch in self.patches.all()],
-                key=lambda x: json.dumps(x, sort_keys=True),
-            ),
-            "severities": sorted(
-                [sev.to_vulnerability_severity_data().to_dict() for sev in self.severities.all()],
-                key=lambda x: (x.get("system"), x.get("value")),
-            ),
-            "weaknesses": normalize_list([weakness.cwe_id for weakness in self.weaknesses.all()]),
-        }
-
-        normalized_json = json.dumps(normalized_data, separators=(",", ":"), sort_keys=True)
-        content_hash = hashlib.sha256(normalized_json.encode("utf-8")).hexdigest()
-
-        return content_hash
-
     alias = get_aliases
 
 

vulnerabilities/pipelines/enhance_with_exploitdb.py

@@ -78,19 +78,16 @@ def add_exploit(self):
 
 
 def add_vulnerability_exploit(row, logger):
-    vulnerabilities = set()
-
     aliases = row["codes"].split(";") if row["codes"] else []
 
     if not aliases:
         return 0
 
-    for raw_alias in aliases:
-        try:
-            if alias := Alias.objects.get(alias=raw_alias):
-                vulnerabilities.add(alias.vulnerability)
-        except Alias.DoesNotExist:
-            continue
+    vulnerabilities = (
+        Alias.objects.filter(alias__in=aliases, vulnerability__isnull=False)
+        .values_list("vulnerability_id", flat=True)
+        .distinct()
+    )
 
     if not vulnerabilities:
         logger(f"No vulnerability found for aliases {aliases}")
@@ -104,7 +101,7 @@ def add_vulnerability_exploit(row, logger):
         add_exploit_references(row["codes"], row["source_url"], row["file"], vulnerability, logger)
         try:
             Exploit.objects.update_or_create(
-                vulnerability=vulnerability,
+                vulnerability_id=vulnerability,
                 data_source="Exploit-DB",
                 defaults={
                     "date_added": date_added,
@@ -125,7 +122,7 @@ def add_vulnerability_exploit(row, logger):
     return 1
 
 
-def add_exploit_references(ref_id, direct_url, path, vul, logger):
+def add_exploit_references(ref_id, direct_url, path, vul_id, logger):
     url_map = {
         "file_url": f"https://gitlab.com/exploit-database/exploitdb/-/blob/main/{path}",
         "direct_url": direct_url,
@@ -144,7 +141,7 @@ def add_exploit_references(ref_id, direct_url, path, vul, logger):
 
                 if created:
                     VulnerabilityRelatedReference.objects.get_or_create(
-                        vulnerability=vul,
+                        vulnerability_id=vul_id,
                         reference=ref,
                     )
 

vulnerabilities/pipelines/enhance_with_kev.py

@@ -71,26 +71,29 @@ def add_vulnerability_exploit(kev_vul, logger):
     if not cve_id:
         return 0
 
-    vulnerability = None
-    try:
-        if alias := Alias.objects.get(alias=cve_id):
-            vulnerability = alias.vulnerability
-    except Alias.DoesNotExist:
+    vulnerabilities = (
+        Alias.objects.filter(alias=cve_id, vulnerability__isnull=False)
+        .values_list("vulnerability", flat=True)
+        .distinct()
+    )
+
+    if not vulnerabilities:
         logger(f"No vulnerability found for aliases {cve_id}")
         return 0
 
-    Exploit.objects.update_or_create(
-        vulnerability=vulnerability,
-        data_source="KEV",
-        defaults={
-            "description": kev_vul["shortDescription"],
-            "date_added": kev_vul["dateAdded"],
-            "required_action": kev_vul["requiredAction"],
-            "due_date": kev_vul["dueDate"],
-            "notes": kev_vul["notes"],
-            "known_ransomware_campaign_use": True
-            if kev_vul["knownRansomwareCampaignUse"] == "Known"
-            else False,
-        },
-    )
+    for vulnerability in vulnerabilities:
+        Exploit.objects.update_or_create(
+            vulnerability_id=vulnerability,
+            data_source="KEV",
+            defaults={
+                "description": kev_vul["shortDescription"],
+                "date_added": kev_vul["dateAdded"],
+                "required_action": kev_vul["requiredAction"],
+                "due_date": kev_vul["dueDate"],
+                "notes": kev_vul["notes"],
+                "known_ransomware_campaign_use": True
+                if kev_vul["knownRansomwareCampaignUse"] == "Known"
+                else False,
+            },
+        )
     return 1

vulnerabilities/pipelines/enhance_with_metasploit.py

@@ -66,7 +66,6 @@ def add_vulnerability_exploits(self):
 
 
 def add_vulnerability_exploit(record, logger):
-    vulnerabilities = set()
     references = record.get("references", [])
 
     interesting_references = [
@@ -76,12 +75,11 @@ def add_vulnerability_exploit(record, logger):
     if not interesting_references:
         return 0
 
-    for ref in interesting_references:
-        try:
-            if alias := Alias.objects.get(alias=ref):
-                vulnerabilities.add(alias.vulnerability)
-        except Alias.DoesNotExist:
-            continue
+    vulnerabilities = (
+        Alias.objects.filter(alias__in=interesting_references, vulnerability__isnull=False)
+        .values_list("vulnerability", flat=True)
+        .distinct()
+    )
 
     if not vulnerabilities:
         logger(f"No vulnerability found for aliases {interesting_references}")
@@ -107,7 +105,7 @@ def add_vulnerability_exploit(record, logger):
 
     for vulnerability in vulnerabilities:
         Exploit.objects.update_or_create(
-            vulnerability=vulnerability,
+            vulnerability_id=vulnerability,
             data_source="Metasploit",
             defaults={
                 "description": description,

vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py

@@ -7,6 +7,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import json
 import logging
 from pathlib import Path
 from typing import Any
@@ -192,7 +193,8 @@ def load_advisories(
 
             fixed_version_range = None
             try:
-                fixed_version_range = AlpineLinuxVersionRange.from_versions([version])
+                if version:
+                    fixed_version_range = AlpineLinuxVersionRange.from_versions([version])
             except InvalidVersion as e:
                 logger(
                     f"{version!r} is not a valid AlpineVersion {e!r}",
@@ -244,4 +246,5 @@ def load_advisories(
                 references=references,
                 affected_packages=affected_packages,
                 url=url,
+                original_advisory_text=json.dumps(pkg_infos, indent=2, ensure_ascii=False),
             )

vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py

@@ -330,19 +330,20 @@ def to_version_ranges(self, versions_data, fixed_versions):
                 "=": "=",
             }
             comparator = comparator_by_range_expression.get(range_expression)
-            if comparator:
+            if comparator and version_value and version_value not in self.ignorable_versions:
                 constraints.append(
                     VersionConstraint(comparator=comparator, version=SemverVersion(version_value))
                 )
 
         for fixed_version in fixed_versions:
             # The VersionConstraint method `invert()` inverts the fixed_version's comparator,
             # enabling inclusion of multiple fixed versions with the `affected_version_range` values.
-            constraints.append(
-                VersionConstraint(
-                    comparator="=",
-                    version=SemverVersion(fixed_version),
-                ).invert()
-            )
+            if fixed_version and fixed_version not in self.ignorable_versions:
+                constraints.append(
+                    VersionConstraint(
+                        comparator="=",
+                        version=SemverVersion(fixed_version),
+                    ).invert()
+                )
 
         return ApacheVersionRange(constraints=constraints)