Merge main and fix conflicts

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -120,6 +120,39 @@ Release notes
   simplicity, and readability.
   https://github.com/aboutcode-org/dejacode/issues/241
 
+- Refine the way the PURL fragments are handled in searches.
+  https://github.com/aboutcode-org/dejacode/issues/286
+
+- Fix an issue with ``urlize_target_blank`` when the URL contains curly braces.
+
+- Add the ability to download Product "Imports" input file.
+  https://github.com/aboutcode-org/dejacode/issues/156
+
+- Fix a logic issue in the ``ImportPackageFromScanCodeIO.import_package`` that occurs when
+  multiple packages with the same PURL, but different download_url or filename,
+  are present in the Dataspace.
+  https://github.com/aboutcode-org/dejacode/issues/295
+
+- Fix a logic issue in the ``ImportPackageFromScanCodeIO.import_dependencies`` to
+  prevent the creation of duplicated "resolved" dependencies.
+  https://github.com/aboutcode-org/dejacode/issues/297
+
+- Display the filename/download_url in the Inventory tab.
+  https://github.com/aboutcode-org/dejacode/issues/303
+
+- Improve exception support in improve_packages_from_purldb task.
+  In case of an exception, the error is properly logged on the Import instance.
+  https://github.com/aboutcode-org/dejacode/issues/303
+
+- Refine the ``update_from_purldb`` function to avoid any IntegrityError.
+  Also, when multiple entries are returned from the PurlDB, only the common values are
+  merged and kept for the data update.
+  https://github.com/aboutcode-org/dejacode/issues/303
+
+- Add a new "Package Set" tab to the Package details view.
+  This tab displays related packages grouped by their normalized ("plain") Package URL.
+  https://github.com/aboutcode-org/dejacode/issues/276
+
 ### Version 5.2.1
 
 - Fix the models documentation navigation.

component_catalog/admin.py

@@ -71,6 +71,7 @@
 from dje.templatetags.dje_tags import urlize_target_blank
 from dje.utils import CHANGELIST_LINK_TEMPLATE
 from dje.utils import get_instance_from_referer
+from dje.utils import is_purl_fragment
 from license_library.models import License
 from reporting.filters import ReportingQueryListFilter
 
@@ -774,7 +775,17 @@ class PackageAdmin(
         "get_dataspace",
     )
     list_display_links = ("identifier",)
-    search_fields = ("filename", "download_url", "project")
+    search_fields = (
+        "type",
+        "namespace",
+        "name",
+        "version",
+        "filename",
+        "download_url",
+        "sha1",
+        "md5",
+        "project",
+    )
     ordering = ("-last_modified_date",)
     list_filter = (
         ("component", HierarchyRelatedLookupListFilter),
@@ -912,6 +923,7 @@ def get_queryset(self, request):
         return (
             super()
             .get_queryset(request)
+            .annotate_sortable_identifier()
             .select_related(
                 "usage_policy",
             )
@@ -938,6 +950,16 @@ def get_urls(self):
 
         return urls + super().get_urls()
 
+    def get_search_results(self, request, queryset, search_term):
+        """Add searching on provided PackageURL identifier."""
+        use_distinct = False
+
+        if is_purl_fragment(search_term):
+            if results := queryset.for_package_url(search_term):
+                return results, use_distinct
+
+        return super().get_search_results(request, queryset, search_term)
+
     def changeform_view(self, request, object_id=None, form_url="", extra_context=None):
         """
         Add the `show_save_and_collect_data` in the context.
@@ -1053,6 +1075,10 @@ def inferred_url(self, obj):
             return urlize_target_blank(inferred_url)
         return ""
 
+    @admin.display(ordering="sortable_identifier")
+    def identifier(self, obj):
+        return obj.identifier
+
     def save_formset(self, request, form, formset, change):
         """
         Update the completion_level on the related Component at the end of the saving process.

component_catalog/filters.py

@@ -22,6 +22,7 @@
 from dje.filters import HasRelationFilter
 from dje.filters import MatchOrderedSearchFilter
 from dje.filters import RelatedLookupListFilter
+from dje.utils import is_purl_fragment
 from dje.widgets import BootstrapSelectMultipleWidget
 from dje.widgets import DropDownRightWidget
 from dje.widgets import SortDropDownWidget
@@ -183,9 +184,9 @@ def filter(self, qs, value):
         if not value:
             return qs
 
-        is_purl = "/" in value
-        if is_purl:
-            return qs.for_package_url(value)
+        if is_purl_fragment(value):
+            if results := qs.for_package_url(value):
+                return results
 
         return super().filter(qs, value)
 

component_catalog/models.py

@@ -18,10 +18,14 @@
 from django.core.exceptions import ValidationError
 from django.core.validators import EMPTY_VALUES
 from django.db import models
+from django.db.models import Case
 from django.db.models import CharField
 from django.db.models import Count
 from django.db.models import Exists
+from django.db.models import F
 from django.db.models import OuterRef
+from django.db.models import Value
+from django.db.models import When
 from django.db.models.functions import Concat
 from django.dispatch import receiver
 from django.template.defaultfilters import filesizeformat
@@ -72,6 +76,7 @@
 from dje.models import ReferenceNotesMixin
 from dje.tasks import logger as tasks_logger
 from dje.utils import is_purl_str
+from dje.utils import merge_common_non_empty_values
 from dje.utils import set_fields_from_object
 from dje.validators import generic_uri_validator
 from dje.validators import validate_url_segment
@@ -1650,6 +1655,65 @@ def __str__(self):
 PACKAGE_URL_FIELDS = ["type", "namespace", "name", "version", "qualifiers", "subpath"]
 
 
+def get_plain_package_url_expression():
+    """
+    Return a Django expression to compute the "PLAIN" Package URL (PURL).
+    Return an empty string if the required `type` or `name` values are missing.
+    """
+    plain_package_url = Concat(
+        Value("pkg:"),
+        F("type"),
+        Case(
+            When(namespace="", then=Value("")),
+            default=Concat(Value("/"), F("namespace")),
+            output_field=CharField(),
+        ),
+        Value("/"),
+        F("name"),
+        Case(
+            When(version="", then=Value("")),
+            default=Concat(Value("@"), F("version")),
+            output_field=CharField(),
+        ),
+        output_field=CharField(),
+    )
+
+    return Case(
+        When(type="", then=Value("")),
+        When(name="", then=Value("")),
+        default=plain_package_url,
+        output_field=CharField(),
+    )
+
+
+def get_package_url_expression():
+    """
+    Return a Django expression to compute the "FULL" Package URL (PURL).
+    Return an empty string if the required `type` or `name` values are missing.
+    """
+    package_url = Concat(
+        get_plain_package_url_expression(),
+        Case(
+            When(qualifiers="", then=Value("")),
+            default=Concat(Value("?"), F("qualifiers")),
+            output_field=CharField(),
+        ),
+        Case(
+            When(subpath="", then=Value("")),
+            default=Concat(Value("#"), F("subpath")),
+            output_field=CharField(),
+        ),
+        output_field=CharField(),
+    )
+
+    return Case(
+        When(type="", then=Value("")),
+        When(name="", then=Value("")),
+        default=package_url,
+        output_field=CharField(),
+    )
+
+
 class PackageQuerySet(PackageURLQuerySetMixin, VulnerabilityQuerySetMixin, DataspacedQuerySet):
     def has_package_url(self):
         """Return objects with Package URL defined."""
@@ -1665,6 +1729,26 @@ def annotate_sortable_identifier(self):
             sortable_identifier=Concat(*PACKAGE_URL_FIELDS, "filename", output_field=CharField())
         )
 
+    def annotate_plain_package_url(self):
+        """
+        Annotate the QuerySet with a computed 'plain' Package URL (PURL).
+
+        This plain PURL is a simplified version that includes only the core fields:
+        `type`, `namespace`, `name`, and `version`. It omits any qualifiers or
+        subpath components, providing a normalized and minimal representation
+        of the Package URL.
+        """
+        return self.annotate(plain_purl=get_plain_package_url_expression())
+
+    def annotate_package_url(self):
+        """
+        Annotate the QuerySet with a fully-computed Package URL (PURL).
+
+        This includes the core PURL fields (`type`, `namespace`, `name`, `version`)
+        as well as any qualifiers and subpath components.
+        """
+        return self.annotate(purl=get_package_url_expression())
+
     def only_rendering_fields(self):
         """Minimum requirements to render a Package element in the UI."""
         return self.only(
@@ -2454,6 +2538,7 @@ def get_purldb_entries(self, user, max_request_call=0, timeout=10):
         is nothing was found.
         """
         payloads = []
+        purldb_entries = []
 
         package_url = self.package_url
         if package_url:
@@ -2468,24 +2553,69 @@ def get_purldb_entries(self, user, max_request_call=0, timeout=10):
             if max_request_call and index >= max_request_call:
                 return
 
-            if packages_data := purldb.find_packages(payload, timeout):
-                return packages_data
+            if purldb_entries := purldb.find_packages(payload, timeout):
+                break
+
+        if not purldb_entries:
+            return []
+
+        # Cleanup the PurlDB entries:
+        # - Packages with different PURL are excluded.
+        if package_url:
+            purldb_entries = [entry for entry in purldb_entries if entry.get("purl") == package_url]
+
+        return purldb_entries
 
     def update_from_purldb(self, user):
         """
-        Find this Package in the PurlDB and update empty fields with PurlDB data
-        when available.
+        Update this Package instance with data from PurlDB.
+
+        - Retrieves matching entries from PurlDB using the given user.
+        - If exactly one match is found, its data is used directly.
+        - If multiple entries are found, only values that are non-empty and
+          common across all entries are merged and used to update the Package.
         """
         purldb_entries = self.get_purldb_entries(user)
         if not purldb_entries:
             return
 
-        package_data = purldb_entries[0]
+        purldb_entries_count = len(purldb_entries)
+        if purldb_entries_count == 1:
+            package_data = purldb_entries[0]
+        else:
+            package_data = merge_common_non_empty_values(purldb_entries)
+
         # The format from PURLDB is "2019-11-18T00:00:00Z"
         if release_date := package_data.get("release_date"):
             package_data["release_date"] = release_date.split("T")[0]
         package_data["license_expression"] = package_data.get("declared_license_expression")
 
+        # Avoid raising an IntegrityError when the values in `package_data` for the
+        # identifier fields already exist on another Package instance.
+        #
+        # This situation can occur when a complete package (with both `purl` and
+        # `download_url`) already exists in the Dataspace, and `update_from_purldb` is
+        # called on a different package that has the same `purl` but no `download_url`.
+        #
+        # If we try to assign the same `download_url` to the second package, it would
+        # violate the unique constraints defined in the Package model (since the
+        # combination of fields must be unique).
+        unique_filters_lookups = {
+            field_name: package_data.get(field_name, "")
+            for field_name in self.get_identifier_fields()
+        }
+        unique_filters_qs = (
+            Package.objects.scope(self.dataspace)
+            .filter(**unique_filters_lookups)
+            .exclude(pk=self.pk)
+        )
+        if unique_filters_qs.exists():
+            # Remove the problematic "identifier_fields" values and the checksum values
+            hash_field_names = [field.name for field in HashFieldsMixin._meta.fields]
+            identifier_fields = self.get_identifier_fields()
+            for field_name in [*hash_field_names, *identifier_fields]:
+                package_data.pop(field_name, None)
+
         updated_fields = self.update_from_data(
             user,
             package_data,
@@ -2508,6 +2638,32 @@ def update_from_scan(self, user):
             updated_fields = scancodeio.update_from_scan(package=self, user=user)
             return updated_fields
 
+    def get_related_packages_qs(self):
+        """
+        Return a QuerySet of packages that are considered part of the same
+        "Package Set".
+
+        A "Package Set" consists of all packages that share the same "plain"
+        Package URL (PURL), meaning they have identical values for the following PURL
+        components:
+        `type`, `namespace`, `name`, and `version`.
+        The `qualifiers` and `subpath` components  are ignored for this comparison.
+        """
+        plain_package_url = self.plain_package_url
+        if not plain_package_url:
+            return None
+
+        return (
+            self.__class__.objects.scope(self.dataspace)
+            .for_package_url(plain_package_url, exact_match=True)
+            .order_by(
+                *PACKAGE_URL_FIELDS,
+                "filename",
+                "download_url",
+            )
+            .distinct()
+        )
+
 
 class PackageAssignedLicense(DataspacedModel):
     package = models.ForeignKey(

component_catalog/templates/component_catalog/includes/package_filename_as_link.html

@@ -0,0 +1,19 @@
+<div title="{{ package.download_url }}"{% if not package.filename %} class="text-truncate"{% endif %}>
+  {% if package.download_url %}
+    <a href="{{ package.download_url }}">
+      {% if display_icons %}
+        <i class="fa-solid fa-download me-1"></i>
+      {% endif %}
+      {% if package.filename %}
+        {{ package.filename }}
+      {% else %}
+        {{ package.download_url|truncatechars:40 }}
+      {% endif %}
+    </a>
+  {% elif package.filename %}
+    {% if display_icons %}
+      <i class="fa-solid fa-file me-1"></i>
+    {% endif %}
+    {{ package.filename }}
+  {% endif %}
+</div>

component_catalog/templates/component_catalog/tables/package_list_table.html

@@ -1,6 +1,5 @@
 {% load i18n %}
 {% load inject_preserved_filters from dje_tags %}
-{% load urlize_target_blank from dje_tags %}
 {% load naturaltime_short from dje_tags %}
 <table id="object-list-table" class="table table-bordered table-striped table-md table-fixed-layout text-break packages-table">
   {% if form or add_to_component_form %}
@@ -58,12 +57,8 @@
       <td>
         {{ object.primary_language }}
       </td>
-      <td title="{{ object.download_url }}"{% if not object.filename %} class="text-truncate"{% endif %}>
-        {% if object.download_url %}
-          <a href="{{ object.download_url }}">
-            {% if object.filename %}{{ object.filename }}{% else %}{{ object.download_url }}{% endif %}
-          </a>
-        {% endif %}
+      <td>
+        {% include 'component_catalog/includes/package_filename_as_link.html' with package=object %}
       </td>
       <td>
         {% with components=object.component_set.all %}