Migrate Advisory aliases field to M2M relationship

Signed-off-by: Keshav Priyadarshi <git@keshav.space>

Author: keshav-space

vulnerabilities/migrations/0089_migrate_advisory_aliases.py

@@ -0,0 +1,117 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from aboutcode.pipeline import LoopProgress
+from django.db import migrations
+from django.db import models
+
+"""
+Model and data migration for converting the Advisory aliases
+JSON field to a concrete M2M Advisory Alias relationship.
+"""
+
+def bulk_update(model, items, fields, logger):
+    item_count = 0
+    if items:
+        try:
+            model.objects.bulk_update(objs=items, fields=fields)
+            item_count += len(items)
+        except Exception as e:
+            logger(f"Error updating Advisory: {e}")
+        items.clear()
+    return item_count
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0088_fix_alpine_purl_type"),
+    ]
+
+    def populate_new_advisory_aliases_field(apps, schema_editor):
+        Advisory = apps.get_model("vulnerabilities", "Advisory")
+        Alias = apps.get_model("vulnerabilities", "Alias")
+        advisories = Advisory.objects.all()
+
+        chunk_size = 10000
+        advisories_count = advisories.count()
+        print(f"\nPopulate new advisory aliases relationship.")
+        progress = LoopProgress(
+            total_iterations=advisories_count,
+            logger=print,
+            progress_step=1,
+        )
+        for advisory in progress.iter(advisories.iterator(chunk_size=chunk_size)):
+            aliases = Alias.objects.filter(alias__in=advisory.old_aliases)
+            advisory.aliases.set(aliases)
+
+    def reverse_populate_new_advisory_aliases_field(apps, schema_editor):
+        Advisory = apps.get_model("vulnerabilities", "Advisory")
+        advisories = Advisory.objects.all()
+
+        updated_advisory_count = 0
+        batch_size = 10000
+        chunk_size = 10000
+        updated_advisory = []
+        progress = LoopProgress(
+            total_iterations=advisories.count(),
+            logger=print,
+            progress_step=1,
+        )
+        for advisory in progress.iter(advisories.iterator(chunk_size=chunk_size)):
+            aliases = advisory.aliases.all()
+            advisory.old_aliases = [alias.alias for alias in aliases]
+            updated_advisory.append(advisory)
+
+            if len(updated_advisory) > batch_size:
+                updated_advisory_count += bulk_update(
+                    model=Advisory,
+                    items=updated_advisory,
+                    fields=["old_aliases"],
+                    logger=print,
+                )
+
+        updated_advisory_count += bulk_update(
+            model=Advisory,
+            items=updated_advisory,
+            fields=["old_aliases"],
+            logger=print,
+        )
+
+    operations = [
+        # Rename aliases field to old_aliases
+        migrations.AlterModelOptions(
+            name="advisory",
+            options={"ordering": ["date_published", "unique_content_id"]},
+        ),
+        migrations.AlterUniqueTogether(
+            name="advisory",
+            unique_together={("unique_content_id", "date_published", "url")},
+        ),
+        migrations.RenameField(
+            model_name="advisory",
+            old_name="aliases",
+            new_name="old_aliases",
+        ),
+        migrations.AddField(
+            model_name="advisory",
+            name="aliases",
+            field=models.ManyToManyField(related_name="advisories", to="vulnerabilities.alias"),
+        ),
+        # Populate the new M2M aliases relation
+        migrations.RunPython(
+            code=populate_new_advisory_aliases_field,
+            reverse_code=reverse_populate_new_advisory_aliases_field,
+        ),
+        # Delete old_alias field
+        migrations.RemoveField(
+            model_name="advisory",
+            name="old_aliases",
+        ),
+    ]

vulnerabilities/models.py

@@ -1318,7 +1318,10 @@ class Advisory(models.Model):
         max_length=32,
         blank=True,
     )
-    aliases = models.JSONField(blank=True, default=list, help_text="A list of alias strings")
+    aliases = models.ManyToManyField(
+        Alias,
+        related_name="advisories",
+    )
     summary = models.TextField(
         blank=True,
     )
@@ -1353,8 +1356,8 @@ class Advisory(models.Model):
     objects = AdvisoryQuerySet.as_manager()
 
     class Meta:
-        unique_together = ["aliases", "unique_content_id", "date_published", "url"]
-        ordering = ["aliases", "date_published", "unique_content_id"]
+        unique_together = ["unique_content_id", "date_published", "url"]
+        ordering = ["date_published", "unique_content_id"]
 
     def save(self, *args, **kwargs):
         checksum = hashlib.md5()

vulnerabilities/tests/test_changelog.py

@@ -17,6 +17,7 @@
 from vulnerabilities import models
 from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.pipelines.npm_importer import NpmImporterPipeline
+from vulnerabilities.pipes.advisory import get_or_create_aliases
 
 
 @pytest.mark.django_db
@@ -37,8 +38,8 @@ def test_package_changelog():
                 fixed_version=SemverVersion("1.0"),
             ).to_dict()
         ],
-        aliases=["CVE-123"],
     )
+    adv.aliases.add(*get_or_create_aliases(["CVE-123"]))
     NpmImporterPipeline().import_advisory(advisory=adv)
     assert models.PackageChangeLog.objects.filter(package=pkg).count() == 1
     NpmImporterPipeline().import_advisory(advisory=adv)
@@ -65,8 +66,8 @@ def test_package_changelog():
                 affected_version_range=NpmVersionRange.from_native(">=2.0"),
             ).to_dict()
         ],
-        aliases=["CVE-145"],
     )
+    adv.aliases.add(*get_or_create_aliases(["CVE-123"]))
     NpmImporterPipeline().import_advisory(advisory=adv)
     assert models.PackageChangeLog.objects.filter(package=pkg1).count() == 1
     NpmImporterPipeline().import_advisory(advisory=adv)
@@ -96,8 +97,8 @@ def test_vulnerability_changelog():
                 fixed_version=SemverVersion("1.0"),
             ).to_dict()
         ],
-        aliases=["CVE-TEST-1234"],
     )
+    adv.aliases.add(*get_or_create_aliases(["CVE-TEST-1234"]))
     NpmImporterPipeline().import_advisory(advisory=adv)
     # 1 Changelogs is expected here:
     # 1 for importing vuln details
@@ -129,8 +130,8 @@ def test_vulnerability_changelog_software_version():
                 fixed_version=SemverVersion("1.0"),
             ).to_dict()
         ],
-        aliases=["CVE-TEST-1234"],
     )
+    adv.aliases.add(*get_or_create_aliases(["CVE-TEST-1234"]))
     NpmImporterPipeline().import_advisory(advisory=adv)
     npm_vulnerability_log = models.VulnerabilityChangeLog.objects.first()