Avoid the IntegrityError for duplicated identifier fields values #303

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

component_catalog/models.py

@@ -17,7 +17,6 @@
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.exceptions import ValidationError
 from django.core.validators import EMPTY_VALUES
-from django.db import IntegrityError
 from django.db import models
 from django.db.models import CharField
 from django.db.models import Count
@@ -2505,16 +2504,42 @@ def update_from_purldb(self, user):
             package_data["release_date"] = release_date.split("T")[0]
         package_data["license_expression"] = package_data.get("declared_license_expression")
 
-        try:
-            updated_fields = self.update_from_data(
-                user,
-                package_data,
-                override=False,
-                override_unknown=True,
-            )
-        except IntegrityError as e:
-            logger.error(f"[update_from_purldb] Skipping {self} due to IntegrityError: {e}")
-            return []
+        # Avoid raising an IntegrityError when the values in `package_data` for the
+        # identifier fields already exist on another Package instance.
+        #
+        # This situation can occur when a complete package (with both `purl` and
+        # `download_url`) already exists in the Dataspace, and `update_from_purldb` is
+        # called on a different package that has the same `purl` but no `download_url`.
+        #
+        # If we try to assign the same `download_url` to the second package, it would
+        # violate the unique constraints defined in the Package model (since the
+        # combination of fields must be unique).
+        unique_filters_lookups = {
+            field_name: package_data.get(field_name, "")
+            for field_name in self.get_identifier_fields()
+        }
+        unique_filters_qs = (
+            Package.objects.scope(self.dataspace)
+            .filter(**unique_filters_lookups)
+            .exclude(pk=self.pk)
+        )
+        if unique_filters_qs.exists():
+            # Remove the problematic "identifier_fields" values and the checksum values
+            hash_field_names = [field.name for field in HashFieldsMixin._meta.fields]
+            identifier_fields = self.get_identifier_fields()
+            for field_name in [*hash_field_names, *identifier_fields]:
+                package_data.pop(field_name, None)
+
+        # try:
+        updated_fields = self.update_from_data(
+            user,
+            package_data,
+            override=False,
+            override_unknown=True,
+        )
+        # except IntegrityError as e:
+        #     logger.error(f"[update_from_purldb] Skipping {self} due to IntegrityError: {e}")
+        #     return []
 
         return updated_fields
 

component_catalog/tests/test_models.py

@@ -2670,16 +2670,25 @@ def test_package_model_update_from_purldb_duplicate_exception(self, mock_get_pur
             "name": "django",
             "version": "3.0",
             "download_url": download_url,
+            "description": "This value will be updated",
+            "md5": "This value is skipped",
+            "sha1": "This value is skipped",
         }
-
         mock_get_purldb_entries.return_value = [purldb_entry]
+
+        # 2 packages with the same "pkg:pypi/django@3.0" PURL:
+        # - 1 with a `download_url` value
+        # - 1 without a `download_url` value
         make_package(self.dataspace, package_url=package_url, download_url=download_url)
         package_no_download_url = make_package(self.dataspace, package_url=package_url)
 
-        # Updating the package with the download_url form purldb_entry would violates the
-        # unique constraint. This is handle properly by update_from_purldb.
+        # Updating the package with the `download_url` from the purldb_entry data
+        # would violates the unique constraint.
+        # This is handle properly by update_from_purldb.
         updated_fields = package_no_download_url.update_from_purldb(self.user)
-        self.assertEqual([], updated_fields)
+        self.assertEqual(["description"], updated_fields)
+        package_no_download_url.refresh_from_db()
+        self.assertEqual(purldb_entry["description"], package_no_download_url.description)
 
     def test_package_model_vulnerability_queryset_mixin(self):
         package1 = make_package(self.dataspace, is_vulnerable=True)