Add option to infer_download_urls on Product importers

Signed-off-by: tdruez <tdruez@aboutcode.org>

Author: tdruez

product_portfolio/forms.py

@@ -641,6 +641,15 @@ class BaseProductImportFormView(forms.Form):
             "for all of the packages assigned to your product."
         ),
     )
+    infer_download_urls = forms.BooleanField(
+        label=_("Infer missing download URLs"),
+        required=False,
+        initial=True,
+        help_text=_(
+            "When a download URL is missing from the input data, attempt to infer it "
+            "from the Package URL (purl). A download URL is required for package scanning."
+        ),
+    )
 
     @property
     def helper(self):
@@ -652,6 +661,7 @@ def helper(self):
             Fieldset(
                 None,
                 "input_file",
+                "infer_download_urls",
                 "update_existing_packages",
                 "scan_all_packages",
                 StrictSubmit("submit", _("Import"), css_class="btn-success col-2"),
@@ -667,6 +677,7 @@ def submit(self, product, user):
             input_file=self.cleaned_data.get("input_file"),
             update_existing_packages=self.cleaned_data.get("update_existing_packages"),
             scan_all_packages=self.cleaned_data.get("scan_all_packages"),
+            infer_download_urls=self.cleaned_data.get("infer_download_urls"),
             created_by=user,
         )
 

product_portfolio/importers.py

@@ -26,6 +26,7 @@
 from component_catalog.models import PACKAGE_URL_FIELDS
 from component_catalog.models import Component
 from component_catalog.models import Package
+from dejacode_toolkit import download
 from dejacode_toolkit.scancodeio import ScanCodeIO
 from dje.copier import copy_object
 from dje.importers import BaseImporter
@@ -649,7 +650,15 @@ class ImportPackageFromScanCodeIO:
         "filename",
     ]
 
-    def __init__(self, user, project_uuid, product, update_existing=False, scan_all_packages=False):
+    def __init__(
+        self,
+        user,
+        project_uuid,
+        product,
+        update_existing=False,
+        scan_all_packages=False,
+        infer_download_urls=False,
+    ):
         self.licensing = Licensing()
         self.created = defaultdict(list)
         self.existing = defaultdict(list)
@@ -662,6 +671,7 @@ def __init__(self, user, project_uuid, product, update_existing=False, scan_all_
         self.product = product
         self.update_existing = update_existing
         self.scan_all_packages = scan_all_packages
+        self.infer_download_urls = infer_download_urls
 
         scancodeio = ScanCodeIO(user.dataspace)
         self.packages = scancodeio.fetch_project_packages(self.project_uuid)
@@ -696,6 +706,15 @@ def import_package(self, package_data):
         # Check if the package already exists to prevent duplication.
         package = self.look_for_existing_package(package_data)
 
+        # Infer a download URL from the Package URL
+        if (
+            self.infer_download_urls
+            and not package_data.get("download_url")
+            and (purl := package_data.get("purl"))
+            and (download_url := download.infer_download_url(purl))
+        ):
+            package_data["download_url"] = download_url
+
         if license_expression := package_data.get("declared_license_expression"):
             license_expression = str(self.licensing.dedup(license_expression))
             package_data["license_expression"] = license_expression

product_portfolio/migrations/0014_scancodeproject_infer_download_urls.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2.8 on 2025-12-16 04:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("product_portfolio", "0013_productstatus_is_locked_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="scancodeproject",
+            name="infer_download_urls",
+            field=models.BooleanField(default=False),
+        ),
+    ]

product_portfolio/models.py

@@ -1555,6 +1555,9 @@ class Status(models.TextChoices):
     scan_all_packages = models.BooleanField(
         default=False,
     )
+    infer_download_urls = models.BooleanField(
+        default=False,
+    )
     status = models.CharField(
         max_length=10,
         choices=Status.choices,
@@ -1615,6 +1618,7 @@ def import_data_from_scancodeio(self):
             product=self.product,
             update_existing=self.update_existing_packages,
             scan_all_packages=self.scan_all_packages,
+            infer_download_urls=self.infer_download_urls,
         )
         created, existing, errors = importer.save()