Migrate Advisory aliases field to M2M relationship

vulnerabilities/import_runner.py

@@ -15,6 +15,7 @@
 
 from django.core.exceptions import ValidationError
 from django.db import transaction
+from django.db.models.query import QuerySet
 
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import Importer
@@ -96,12 +97,17 @@ def process_advisories(
         Insert advisories into the database
         Return the number of inserted advisories.
         """
+        from vulnerabilities.pipes.advisory import get_or_create_aliases
+        from vulnerabilities.utils import compute_content_id
+
         count = 0
         advisories = []
         for data in advisory_datas:
+            content_id = compute_content_id(advisory_data=data)
             try:
+                aliases = get_or_create_aliases(aliases=data.aliases)
                 obj, created = Advisory.objects.get_or_create(
-                    aliases=data.aliases,
+                    unique_content_id=content_id,
                     summary=data.summary,
                     affected_packages=[pkg.to_dict() for pkg in data.affected_packages],
                     references=[ref.to_dict() for ref in data.references],
@@ -113,6 +119,7 @@ def process_advisories(
                     },
                     url=data.url,
                 )
+                obj.aliases.add(*aliases)
                 if not obj.date_imported:
                     advisories.append(obj)
             except Exception as e:
@@ -148,6 +155,8 @@ def process_inferences(inferences: List[Inference], advisory: Advisory, improver
     erroneous. Also, the atomic transaction for every advisory and its
     inferences makes sure that date_imported of advisory is consistent.
     """
+    from vulnerabilities.pipes.advisory import get_or_create_aliases
+
     inferences_processed_count = 0
 
     if not inferences:
@@ -157,9 +166,10 @@ def process_inferences(inferences: List[Inference], advisory: Advisory, improver
     logger.info(f"Improving advisory id: {advisory.id}")
 
     for inference in inferences:
+        aliases = get_or_create_aliases(inference.aliases)
         vulnerability = get_or_create_vulnerability_and_aliases(
             vulnerability_id=inference.vulnerability_id,
-            aliases=inference.aliases,
+            aliases=aliases,
             summary=inference.summary,
             advisory=advisory,
         )
@@ -265,14 +275,13 @@ def create_valid_vulnerability_reference(url, reference_id=None):
 
 
 def get_or_create_vulnerability_and_aliases(
-    aliases: List[str], vulnerability_id=None, summary=None, advisory=None
+    aliases: QuerySet, vulnerability_id=None, summary=None, advisory=None
 ):
     """
     Get or create vulnerabilitiy and aliases such that all existing and new
     aliases point to the same vulnerability
     """
-    aliases = set(alias.strip() for alias in aliases if alias and alias.strip())
-    new_alias_names, existing_vulns = get_vulns_for_aliases_and_get_new_aliases(aliases)
+    new_aliases, existing_vulns = get_vulns_for_aliases_and_get_new_aliases(aliases)
 
     # All aliases must point to the same vulnerability
     vulnerability = None
@@ -310,11 +319,11 @@ def get_or_create_vulnerability_and_aliases(
         #         f"Inconsistent summary for {vulnerability.vulnerability_id}. "
         #         f"Existing: {vulnerability.summary!r}, provided: {summary!r}"
         #     )
-        associate_vulnerability_with_aliases(vulnerability=vulnerability, aliases=new_alias_names)
+        associate_vulnerability_with_aliases(vulnerability=vulnerability, aliases=new_aliases)
     else:
         try:
             vulnerability = create_vulnerability_and_add_aliases(
-                aliases=new_alias_names, summary=summary
+                aliases=new_aliases, summary=summary
             )
             importer_name = get_importer_name(advisory)
             VulnerabilityChangeLog.log_import(
@@ -324,24 +333,22 @@ def get_or_create_vulnerability_and_aliases(
             )
         except Exception as e:
             logger.error(
-                f"Cannot create vulnerability with summary {summary!r} and {new_alias_names!r} {e!r}.\n{traceback_format_exc()}."
+                f"Cannot create vulnerability with summary {summary!r} and {new_aliases!r} {e!r}.\n{traceback_format_exc()}."
             )
             return
 
     return vulnerability
 
 
-def get_vulns_for_aliases_and_get_new_aliases(aliases):
+def get_vulns_for_aliases_and_get_new_aliases(aliases: QuerySet):
     """
     Return ``new_aliases`` that are not in the database and
     ``existing_vulns`` that point to the given ``aliases``.
     """
-    new_aliases = set(aliases)
-    existing_vulns = set()
-    for alias in Alias.objects.filter(alias__in=aliases):
-        existing_vulns.add(alias.vulnerability)
-        new_aliases.remove(alias.alias)
-    return new_aliases, existing_vulns
+    new_aliases = aliases.filter(vulnerability__isnull=True)
+    existing_vulns = [alias.vulnerability for alias in aliases.filter(vulnerability__isnull=False)]
+
+    return new_aliases, list(set(existing_vulns))
 
 
 @transaction.atomic
@@ -360,7 +367,5 @@ def create_vulnerability_and_add_aliases(aliases, summary):
 
 
 def associate_vulnerability_with_aliases(aliases, vulnerability):
-    for alias_name in aliases:
-        alias = Alias(alias=alias_name, vulnerability=vulnerability)
-        alias.save()
-        logger.info(f"New alias for {vulnerability!r}: {alias_name}")
+    aliases.update(vulnerability=vulnerability)
+    logger.info(f"New alias for {vulnerability!r}: {aliases}")

vulnerabilities/improvers/vulnerability_status.py

@@ -37,11 +37,7 @@ class VulnerabilityStatusImprover(Improver):
 
     @property
     def interesting_advisories(self) -> QuerySet:
-        return (
-            Advisory.objects.filter(Q(created_by=NVDImporterPipeline.pipeline_id))
-            .distinct("aliases")
-            .paginated()
-        )
+        return Advisory.objects.filter(Q(created_by=NVDImporterPipeline.pipeline_id)).paginated()
 
     def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]:
         """

vulnerabilities/migrations/0090_migrate_advisory_aliases.py

@@ -0,0 +1,142 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from aboutcode.pipeline import LoopProgress
+from django.db import migrations
+from django.db import models
+import django.db.models.deletion
+
+"""
+Model and data migration for converting the Advisory aliases
+JSON field to a concrete M2M Advisory Alias relationship.
+"""
+
+def bulk_update(model, items, fields, logger):
+    item_count = 0
+    if items:
+        try:
+            model.objects.bulk_update(objs=items, fields=fields)
+            item_count += len(items)
+        except Exception as e:
+            logger(f"Error updating Advisory: {e}")
+        items.clear()
+    return item_count
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0089_alter_advisory_unique_content_id"),
+    ]
+
+    def populate_new_advisory_aliases_field(apps, schema_editor):
+        Advisory = apps.get_model("vulnerabilities", "Advisory")
+        Alias = apps.get_model("vulnerabilities", "Alias")
+        advisories = Advisory.objects.all()
+
+        chunk_size = 10000
+        advisories_count = advisories.count()
+        print(f"\nPopulate new advisory aliases relationship.")
+        progress = LoopProgress(
+            total_iterations=advisories_count,
+            logger=print,
+            progress_step=1,
+        )
+        for advisory in progress.iter(advisories.iterator(chunk_size=chunk_size)):
+            aliases = Alias.objects.filter(alias__in=advisory.old_aliases)
+            advisory.aliases.set(aliases)
+
+    def reverse_populate_new_advisory_aliases_field(apps, schema_editor):
+        Advisory = apps.get_model("vulnerabilities", "Advisory")
+        advisories = Advisory.objects.all()
+
+        updated_advisory_count = 0
+        batch_size = 10000
+        chunk_size = 10000
+        updated_advisory = []
+        progress = LoopProgress(
+            total_iterations=advisories.count(),
+            logger=print,
+            progress_step=1,
+        )
+        for advisory in progress.iter(advisories.iterator(chunk_size=chunk_size)):
+            aliases = advisory.aliases.all()
+            advisory.old_aliases = [alias.alias for alias in aliases]
+            updated_advisory.append(advisory)
+
+            if len(updated_advisory) > batch_size:
+                updated_advisory_count += bulk_update(
+                    model=Advisory,
+                    items=updated_advisory,
+                    fields=["old_aliases"],
+                    logger=print,
+                )
+
+        updated_advisory_count += bulk_update(
+            model=Advisory,
+            items=updated_advisory,
+            fields=["old_aliases"],
+            logger=print,
+        )
+
+    operations = [
+        migrations.AlterField(
+            model_name="advisory",
+            name="unique_content_id",
+            field=models.CharField(
+                help_text="A 64 character unique identifier for the content of the advisory since we use sha256 as hex",
+                max_length=64,
+                blank=False,
+                null=False,
+            ),
+        ),
+
+        # Make vulnerability relation optional
+        migrations.AlterField(
+            model_name="alias",
+            name="vulnerability",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="aliases",
+                to="vulnerabilities.vulnerability",
+            ),
+        ),
+
+        # Rename aliases field to old_aliases
+        migrations.AlterModelOptions(
+            name="advisory",
+            options={"ordering": ["date_published", "unique_content_id"]},
+        ),
+        migrations.AlterUniqueTogether(
+            name="advisory",
+            unique_together={("unique_content_id", "date_published", "url")},
+        ),
+        migrations.RenameField(
+            model_name="advisory",
+            old_name="aliases",
+            new_name="old_aliases",
+        ),
+        migrations.AddField(
+            model_name="advisory",
+            name="aliases",
+            field=models.ManyToManyField(related_name="advisories", to="vulnerabilities.alias"),
+        ),
+        # Populate the new M2M aliases relation
+        migrations.RunPython(
+            code=populate_new_advisory_aliases_field,
+            reverse_code=reverse_populate_new_advisory_aliases_field,
+        ),
+        # Delete JSON aliases field
+        migrations.RemoveField(
+            model_name="advisory",
+            name="old_aliases",
+        ),
+    ]

vulnerabilities/models.py

@@ -1276,8 +1276,10 @@ class Alias(models.Model):
 
     vulnerability = models.ForeignKey(
         Vulnerability,
-        on_delete=models.CASCADE,
         related_name="aliases",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
     )
 
     objects = AliasQuerySet.as_manager()
@@ -1317,10 +1319,14 @@ class Advisory(models.Model):
 
     unique_content_id = models.CharField(
         max_length=64,
-        blank=True,
+        blank=False,
+        null=False,
         help_text="A 64 character unique identifier for the content of the advisory since we use sha256 as hex",
     )
-    aliases = models.JSONField(blank=True, default=list, help_text="A list of alias strings")
+    aliases = models.ManyToManyField(
+        Alias,
+        related_name="advisories",
+    )
     summary = models.TextField(
         blank=True,
     )
@@ -1355,21 +1361,20 @@ class Advisory(models.Model):
     objects = AdvisoryQuerySet.as_manager()
 
     class Meta:
-        unique_together = ["aliases", "unique_content_id", "date_published", "url"]
-        ordering = ["aliases", "date_published", "unique_content_id"]
+        unique_together = ["unique_content_id", "date_published", "url"]
+        ordering = ["date_published", "unique_content_id"]
 
     def save(self, *args, **kwargs):
-        advisory_data = self.to_advisory_data()
-        self.unique_content_id = compute_content_id(advisory_data, include_metadata=False)
-        super().save(*args, **kwargs)
+        self.full_clean()
+        return super().save(*args, **kwargs)
 
     def to_advisory_data(self) -> "AdvisoryData":
         from vulnerabilities.importer import AdvisoryData
         from vulnerabilities.importer import AffectedPackage
         from vulnerabilities.importer import Reference
 
         return AdvisoryData(
-            aliases=self.aliases,
+            aliases=[item.alias for item in self.aliases.all()],
             summary=self.summary,
             affected_packages=[
                 AffectedPackage.from_dict(pkg) for pkg in self.affected_packages if pkg

vulnerabilities/pipelines/add_cvss31_to_CVEs.py

@@ -12,6 +12,7 @@
 
 from vulnerabilities import severity_systems
 from vulnerabilities.models import Advisory
+from vulnerabilities.models import Alias
 from vulnerabilities.models import VulnerabilitySeverity
 from vulnerabilities.pipelines import VulnerableCodePipeline
 
@@ -48,7 +49,6 @@ def process_cve_advisory_mapping(self):
         results = []
 
         for severity in progress.iter(nvd_severities.paginated(per_page=batch_size)):
-            print(severity.url)
             cve_pattern = re.compile(r"(CVE-\d{4}-\d{4,7})").search
             cve_match = cve_pattern(severity.url)
             if cve_match:
@@ -57,12 +57,10 @@ def process_cve_advisory_mapping(self):
                 self.log(f"Could not find CVE ID in URL: {severity.url}")
                 continue
 
-            matching_advisories = Advisory.objects.filter(
-                aliases=[cve_id],
-                created_by="nvd_importer",
-            )
+            if matching_alias := Alias.objects.get(alias=cve_id):
+                matching_advisories = matching_alias.advisories.filter(created_by="nvd_importer")
 
-            for advisory in matching_advisories:
+            for advisory in matching_advisories or []:
                 for reference in advisory.references:
                     for sev in reference.get("severities", []):
                         if sev.get("system") == "cvssv3.1":
@@ -76,7 +74,6 @@ def process_cve_advisory_mapping(self):
                             )
 
         if results:
-            print(results)
             self._process_batch(results)
 
         self.log(f"Completed processing CVE to Advisory mappings")