Merge branch 'main' into feature/update-workflow

Author: TG1999

CHANGELOG.rst

@@ -2,6 +2,27 @@ Release notes
 =============
 
 
+Version v36.0.0
+---------------------
+
+- Add indexes for models https://github.com/aboutcode-org/vulnerablecode/pull/1701
+- Add fixed by package in V2 API https://github.com/aboutcode-org/vulnerablecode/pull/1706
+- Add tests for num queries for views https://github.com/aboutcode-org/vulnerablecode/pull/1730
+- Add postgresql conf in docker-compose https://github.com/aboutcode-org/vulnerablecode/pull/1733
+- Add default postgresql.conf for local docker build https://github.com/aboutcode-org/vulnerablecode/pull/1735
+- Add models for CodeFix https://github.com/aboutcode-org/vulnerablecode/pull/1704
+- Migrate Alpine Linux importer to aboutcode pipeline https://github.com/aboutcode-org/vulnerablecode/pull/1737
+- VCIO-next: Allow CVSS3.1 Severities in NVD https://github.com/aboutcode-org/vulnerablecode/pull/1738
+- Add Pipeline to add missing CVSSV3.1 scores https://github.com/aboutcode-org/vulnerablecode/pull/1740
+- Add description and reference to the latest release on the homepage https://github.com/aboutcode-org/vulnerablecode/pull/1743
+- Use proper apk package type for Alpine https://github.com/aboutcode-org/vulnerablecode/pull/1739
+- Optimize vulnerabilities view https://github.com/aboutcode-org/vulnerablecode/pull/1728
+- Add CWE support in multiple importers https://github.com/aboutcode-org/vulnerablecode/pull/1526
+- Fast content ID migration https://github.com/aboutcode-org/vulnerablecode/pull/1795
+- Add captcha for user signup https://github.com/aboutcode-org/vulnerablecode/pull/1822
+- Move the package search box to the top by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1832
+
+
 Version v35.1.0
 ---------------------
 

requirements.txt

@@ -20,19 +20,20 @@ charset-normalizer==2.0.12
 click==8.1.2
 coreapi==2.3.3
 coreschema==0.0.4
-cryptography==43.0.1
+cryptography==44.0.1
 crispy-bootstrap4==2024.1
 cvss
 cwe2==3.0.0
 dateparser==1.1.1
 decorator==5.1.1
 defusedxml==0.7.1
 distro==1.7.0
-Django==4.2.17
+Django==4.2.20
 django-crispy-forms==2.3
 django-environ==0.11.2
 django-extensions==3.2.3
 django-filter==24.3
+django-recaptcha==4.0.0
 django-widget-tweaks==1.5.0
 djangorestframework==3.15.2
 doc8==0.11.1
@@ -56,7 +57,7 @@ ipython==8.10.0
 isort==5.10.1
 itypes==1.2.0
 jedi==0.18.1
-Jinja2==3.1.5
+Jinja2==3.1.6
 jsonschema==3.2.0
 license-expression==30.3.1
 lxml==4.9.1

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = vulnerablecode
-version = 35.1.0
+version = 36.0.0
 license = Apache-2.0 AND CC-BY-SA-4.0
 
 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390
@@ -99,6 +99,8 @@ install_requires =
     python-dotenv
     texttable
 
+    django-recaptcha>=4.0.0
+
 
 [options.extras_require]
 dev =

vulnerabilities/forms.py

@@ -9,6 +9,8 @@
 
 from django import forms
 from django.core.validators import validate_email
+from django_recaptcha.fields import ReCaptchaField
+from django_recaptcha.widgets import ReCaptchaV2Checkbox
 
 from vulnerabilities.models import ApiUser
 
@@ -38,6 +40,10 @@ class ApiUserCreationForm(forms.ModelForm):
     Support a simplified creation for API-only users directly from the UI.
     """
 
+    captcha = ReCaptchaField(
+        error_messages={"required": ("Captcha is required")}, widget=ReCaptchaV2Checkbox
+    )
+
     class Meta:
         model = ApiUser
         fields = (

vulnerabilities/import_runner.py

@@ -15,6 +15,7 @@
 
 from django.core.exceptions import ValidationError
 from django.db import transaction
+from django.db.models.query import QuerySet
 
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import Importer
@@ -96,23 +97,29 @@ def process_advisories(
         Insert advisories into the database
         Return the number of inserted advisories.
         """
+        from vulnerabilities.pipes.advisory import get_or_create_aliases
+        from vulnerabilities.utils import compute_content_id
+
         count = 0
         advisories = []
         for data in advisory_datas:
+            content_id = compute_content_id(advisory_data=data)
             try:
+                aliases = get_or_create_aliases(aliases=data.aliases)
                 obj, created = Advisory.objects.get_or_create(
-                    aliases=data.aliases,
-                    summary=data.summary,
-                    affected_packages=[pkg.to_dict() for pkg in data.affected_packages],
-                    references=[ref.to_dict() for ref in data.references],
-                    date_published=data.date_published,
-                    weaknesses=data.weaknesses,
+                    unique_content_id=content_id,
+                    url=data.url,
                     defaults={
+                        "summary": data.summary,
+                        "affected_packages": [pkg.to_dict() for pkg in data.affected_packages],
+                        "references": [ref.to_dict() for ref in data.references],
+                        "date_published": data.date_published,
+                        "weaknesses": data.weaknesses,
                         "created_by": importer_name,
                         "date_collected": datetime.datetime.now(tz=datetime.timezone.utc),
                     },
-                    url=data.url,
                 )
+                obj.aliases.add(*aliases)
                 if not obj.date_imported:
                     advisories.append(obj)
             except Exception as e:
@@ -148,6 +155,8 @@ def process_inferences(inferences: List[Inference], advisory: Advisory, improver
     erroneous. Also, the atomic transaction for every advisory and its
     inferences makes sure that date_imported of advisory is consistent.
     """
+    from vulnerabilities.pipes.advisory import get_or_create_aliases
+
     inferences_processed_count = 0
 
     if not inferences:
@@ -157,9 +166,10 @@ def process_inferences(inferences: List[Inference], advisory: Advisory, improver
     logger.info(f"Improving advisory id: {advisory.id}")
 
     for inference in inferences:
+        aliases = get_or_create_aliases(inference.aliases)
         vulnerability = get_or_create_vulnerability_and_aliases(
             vulnerability_id=inference.vulnerability_id,
-            aliases=inference.aliases,
+            aliases=aliases,
             summary=inference.summary,
             advisory=advisory,
         )
@@ -265,14 +275,13 @@ def create_valid_vulnerability_reference(url, reference_id=None):
 
 
 def get_or_create_vulnerability_and_aliases(
-    aliases: List[str], vulnerability_id=None, summary=None, advisory=None
+    aliases: QuerySet, vulnerability_id=None, summary=None, advisory=None
 ):
     """
     Get or create vulnerabilitiy and aliases such that all existing and new
     aliases point to the same vulnerability
     """
-    aliases = set(alias.strip() for alias in aliases if alias and alias.strip())
-    new_alias_names, existing_vulns = get_vulns_for_aliases_and_get_new_aliases(aliases)
+    new_aliases, existing_vulns = get_vulns_for_aliases_and_get_new_aliases(aliases)
 
     # All aliases must point to the same vulnerability
     vulnerability = None
@@ -310,11 +319,11 @@ def get_or_create_vulnerability_and_aliases(
         #         f"Inconsistent summary for {vulnerability.vulnerability_id}. "
         #         f"Existing: {vulnerability.summary!r}, provided: {summary!r}"
         #     )
-        associate_vulnerability_with_aliases(vulnerability=vulnerability, aliases=new_alias_names)
+        associate_vulnerability_with_aliases(vulnerability=vulnerability, aliases=new_aliases)
     else:
         try:
             vulnerability = create_vulnerability_and_add_aliases(
-                aliases=new_alias_names, summary=summary
+                aliases=new_aliases, summary=summary
             )
             importer_name = get_importer_name(advisory)
             VulnerabilityChangeLog.log_import(
@@ -324,24 +333,22 @@ def get_or_create_vulnerability_and_aliases(
             )
         except Exception as e:
             logger.error(
-                f"Cannot create vulnerability with summary {summary!r} and {new_alias_names!r} {e!r}.\n{traceback_format_exc()}."
+                f"Cannot create vulnerability with summary {summary!r} and {new_aliases!r} {e!r}.\n{traceback_format_exc()}."
             )
             return
 
     return vulnerability
 
 
-def get_vulns_for_aliases_and_get_new_aliases(aliases):
+def get_vulns_for_aliases_and_get_new_aliases(aliases: QuerySet):
     """
     Return ``new_aliases`` that are not in the database and
     ``existing_vulns`` that point to the given ``aliases``.
     """
-    new_aliases = set(aliases)
-    existing_vulns = set()
-    for alias in Alias.objects.filter(alias__in=aliases):
-        existing_vulns.add(alias.vulnerability)
-        new_aliases.remove(alias.alias)
-    return new_aliases, existing_vulns
+    new_aliases = aliases.filter(vulnerability__isnull=True)
+    existing_vulns = [alias.vulnerability for alias in aliases.filter(vulnerability__isnull=False)]
+
+    return new_aliases, list(set(existing_vulns))
 
 
 @transaction.atomic
@@ -360,7 +367,5 @@ def create_vulnerability_and_add_aliases(aliases, summary):
 
 
 def associate_vulnerability_with_aliases(aliases, vulnerability):
-    for alias_name in aliases:
-        alias = Alias(alias=alias_name, vulnerability=vulnerability)
-        alias.save()
-        logger.info(f"New alias for {vulnerability!r}: {alias_name}")
+    aliases.update(vulnerability=vulnerability)
+    logger.info(f"New alias for {vulnerability!r}: {aliases}")

vulnerabilities/importer.py

@@ -9,6 +9,7 @@
 
 import dataclasses
 import datetime
+import functools
 import logging
 import os
 import shutil
@@ -46,7 +47,8 @@
 logger = logging.getLogger(__name__)
 
 
-@dataclasses.dataclass(order=True)
+@dataclasses.dataclass(eq=True)
+@functools.total_ordering
 class VulnerabilitySeverity:
     # FIXME: this should be named scoring_system, like in the model
     system: ScoringSystem
@@ -55,15 +57,26 @@ class VulnerabilitySeverity:
     published_at: Optional[datetime.datetime] = None
 
     def to_dict(self):
-        published_at_dict = (
-            {"published_at": self.published_at.isoformat()} if self.published_at else {}
-        )
-        return {
+        data = {
             "system": self.system.identifier,
             "value": self.value,
             "scoring_elements": self.scoring_elements,
-            **published_at_dict,
         }
+        if self.published_at:
+            if isinstance(self.published_at, datetime.datetime):
+                data["published_at"] = self.published_at.isoformat()
+            else:
+                data["published_at"] = self.published_at
+        return data
+
+    def __lt__(self, other):
+        if not isinstance(other, VulnerabilitySeverity):
+            return NotImplemented
+        return self._cmp_key() < other._cmp_key()
+
+    # TODO: Add cache
+    def _cmp_key(self):
+        return (self.system.identifier, self.value, self.scoring_elements, self.published_at)
 
     @classmethod
     def from_dict(cls, severity: dict):
@@ -79,7 +92,8 @@ def from_dict(cls, severity: dict):
         )
 
 
-@dataclasses.dataclass(order=True)
+@dataclasses.dataclass(eq=True)
+@functools.total_ordering
 class Reference:
     reference_id: str = ""
     reference_type: str = ""
@@ -89,28 +103,31 @@ class Reference:
     def __post_init__(self):
         if not self.url:
             raise TypeError("Reference must have a url")
+        if self.reference_id and not isinstance(self.reference_id, str):
+            self.reference_id = str(self.reference_id)
 
-    def normalized(self):
-        severities = sorted(self.severities)
-        return Reference(
-            reference_id=self.reference_id,
-            url=self.url,
-            severities=severities,
-            reference_type=self.reference_type,
-        )
+    def __lt__(self, other):
+        if not isinstance(other, Reference):
+            return NotImplemented
+        return self._cmp_key() < other._cmp_key()
+
+    # TODO: Add cache
+    def _cmp_key(self):
+        return (self.reference_id, self.reference_type, self.url, tuple(self.severities))
 
     def to_dict(self):
+        """Return a normalized dictionary representation"""
         return {
             "reference_id": self.reference_id,
             "reference_type": self.reference_type,
             "url": self.url,
-            "severities": [severity.to_dict() for severity in self.severities],
+            "severities": [severity.to_dict() for severity in sorted(self.severities)],
         }
 
     @classmethod
     def from_dict(cls, ref: dict):
         return cls(
-            reference_id=ref["reference_id"],
+            reference_id=str(ref["reference_id"]),
             reference_type=ref.get("reference_type") or "",
             url=ref["url"],
             severities=[
@@ -140,7 +157,8 @@ class NoAffectedPackages(Exception):
     """
 
 
-@dataclasses.dataclass(order=True, frozen=True)
+@functools.total_ordering
+@dataclasses.dataclass(eq=True)
 class AffectedPackage:
     """
     Relate a Package URL with a range of affected versions and a fixed version.
@@ -170,6 +188,19 @@ def get_fixed_purl(self):
             raise ValueError(f"Affected Package {self.package!r} does not have a fixed version")
         return update_purl_version(purl=self.package, version=str(self.fixed_version))
 
+    def __lt__(self, other):
+        if not isinstance(other, AffectedPackage):
+            return NotImplemented
+        return self._cmp_key() < other._cmp_key()
+
+    # TODO: Add cache
+    def _cmp_key(self):
+        return (
+            str(self.package),
+            str(self.affected_version_range or ""),
+            str(self.fixed_version or ""),
+        )
+
     @classmethod
     def merge(
         cls, affected_packages: Iterable