Refine update_from_purldb merging data when multiple PURLDB entries #303

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

component_catalog/models.py

@@ -73,6 +73,7 @@
 from dje.models import ReferenceNotesMixin
 from dje.tasks import logger as tasks_logger
 from dje.utils import is_purl_str
+from dje.utils import merge_common_non_empty_values
 from dje.utils import set_fields_from_object
 from dje.validators import generic_uri_validator
 from dje.validators import validate_url_segment
@@ -2482,14 +2483,23 @@ def get_purldb_entries(self, user, max_request_call=0, timeout=10):
 
     def update_from_purldb(self, user):
         """
-        Find this Package in the PurlDB and update empty fields with PurlDB data
-        when available.
+        Update this Package instance with data from PurlDB.
+
+        - Retrieves matching entries from PurlDB using the given user.
+        - If exactly one match is found, its data is used directly.
+        - If multiple entries are found, only values that are non-empty and
+          common across all entries are merged and used to update the Package.
         """
         purldb_entries = self.get_purldb_entries(user)
         if not purldb_entries:
             return
 
-        package_data = purldb_entries[0]
+        purldb_entries_count = len(purldb_entries)
+        if purldb_entries_count == 1:
+            package_data = purldb_entries[0]
+        else:
+            package_data = merge_common_non_empty_values(purldb_entries)
+
         # The format from PURLDB is "2019-11-18T00:00:00Z"
         if release_date := package_data.get("release_date"):
             package_data["release_date"] = release_date.split("T")[0]

component_catalog/tests/test_models.py

@@ -2598,9 +2598,9 @@ def test_package_model_update_from_purldb(self, mock_get_purldb_entries):
         }
 
         mock_get_purldb_entries.return_value = [purldb_entry]
-        package1 = Package.objects.create(
+        package1 = make_package(
+            self.dataspace,
             filename="package",
-            dataspace=self.dataspace,
             # "unknown" values are overrided
             declared_license_expression="unknown",
         )
@@ -2628,6 +2628,38 @@ def test_package_model_update_from_purldb(self, mock_get_purldb_entries):
         for field_name in updated_fields:
             self.assertEqual(purldb_entry[field_name], getattr(package1, field_name))
 
+    @mock.patch("component_catalog.models.Package.get_purldb_entries")
+    def test_package_model_update_from_purldb_multiple_entries(self, mock_get_purldb_entries):
+        purldb_entry1 = {
+            "uuid": "326aa7a8-4f28-406d-89f9-c1404916925b",
+            "purl": "pkg:pypi/django@3.0",
+            "type": "pypi",
+            "name": "django",
+            "version": "3.0",
+            "keywords": ["Keyword1", "Keyword2"],
+            "filename": "Django-3.0.tar.gz",
+            "download_url": "https://files.pythonhosted.org/packages/38/Django-3.0.tar.gz",
+        }
+        purldb_entry2 = {
+            "uuid": "e133e70b-8dd3-4cf1-9711-72b1f57523a0",
+            "purl": "pkg:pypi/django@3.0",
+            "type": "pypi",
+            "name": "django",
+            "version": "3.0",
+            "primary_language": "Python",
+            "keywords": ["Keyword1", "Keyword2"],
+            "download_url": "https://another.url/Django-3.0.tar.gz",
+        }
+
+        mock_get_purldb_entries.return_value = [purldb_entry1, purldb_entry2]
+        package1 = make_package(self.dataspace, package_url="pkg:pypi/django@3.0")
+        updated_fields = package1.update_from_purldb(self.user)
+        expected = ["filename", "keywords", "primary_language"]
+        self.assertEqual(expected, sorted(updated_fields))
+        self.assertEqual("Django-3.0.tar.gz", package1.filename)
+        self.assertEqual(["Keyword1", "Keyword2"], package1.keywords)
+        self.assertEqual("Python", package1.primary_language)
+
     @mock.patch("component_catalog.models.Package.get_purldb_entries")
     def test_package_model_update_from_purldb_duplicate_exception(self, mock_get_purldb_entries):
         package_url = "pkg:pypi/django@3.0"

dje/tests/test_utils.py

@@ -38,6 +38,7 @@
 from dje.utils import is_purl_fragment
 from dje.utils import is_purl_str
 from dje.utils import localized_datetime
+from dje.utils import merge_common_non_empty_values
 from dje.utils import merge_relations
 from dje.utils import normalize_newlines_as_CR_plus_LF
 from dje.utils import remove_field_from_query_dict
@@ -539,3 +540,32 @@ def test_utils_localized_datetime(self):
         self.assertEqual("Jan 13, 2025, 11:11 AM PST", localized_datetime(dt))
         dt = "2025-01-13T19:11:08.216188+01:00"
         self.assertEqual("Jan 13, 2025, 10:11 AM PST", localized_datetime(dt))
+
+    def test_utils_merge_common_non_empty_values(self):
+        entry1 = {
+            "name": "django",
+            "version": "3.0",
+            "description": "A web framework",  # present
+            "uuid": "1234",  # different
+            "empty_field": "",  # empty
+            "missing_in_other": "value",
+            "keywords": ["Keyword1", "Keyword2"],
+            "parties": ["a", "b"],
+        }
+        entry2 = {
+            "name": "django",  # same
+            "version": "3.0",  # same
+            "description": "",  # empty
+            "uuid": "5678",  # different → excluded
+            # "missing_in_other" is missing
+            "keywords": ["Keyword1"],
+            "parties": ["a", "b"],
+        }
+        expected = {
+            "name": "django",
+            "version": "3.0",
+            "description": "A web framework",
+            "missing_in_other": "value",
+            "parties": ["a", "b"],
+        }
+        self.assertEqual(expected, merge_common_non_empty_values([entry1, entry2]))

dje/utils.py

@@ -699,3 +699,23 @@ def localized_datetime(datetime):
 
     default_format = get_format("DATETIME_FORMAT")
     return date_format(dt, default_format)
+
+
+def merge_common_non_empty_values(dicts):
+    """
+    Merge a list of dictionaries by extracting only the key-value pairs
+    that are common and non-empty across all dictionaries.
+    Missing keys are treated as empty values.
+    """
+    merged_result = {}
+    # Collect all unique keys from all dictionaries
+    all_keys = set().union(*dicts)
+
+    for key in all_keys:
+        values = [value for entry in dicts if (value := entry.get(key)) not in EMPTY_VALUES]
+
+        # Include key only if all values are identical
+        if values and all(value == values[0] for value in values):
+            merged_result[key] = values[0]
+
+    return merged_result