Add the infer_download_urls in scan_all_packages_task

Signed-off-by: tdruez <tdruez@aboutcode.org>

Author: tdruez

component_catalog/models.py

@@ -1719,6 +1719,10 @@ def has_package_url(self):
         """Return objects with Package URL defined."""
         return self.filter(~models.Q(type="") & ~models.Q(name=""))
 
+    def has_download_url(self):
+        """Return objects with download URL defined."""
+        return self.filter(~models.Q(download_url=""))
+
     def annotate_sortable_identifier(self):
         """
         Annotate the QuerySet with a `sortable_identifier` value that combines
@@ -2036,9 +2040,13 @@ def package_url_filename(self):
 
     @property
     def inferred_repo_url(self):
-        """Return the URL deduced from the information available in a Package URL (purl)."""
+        """Return the repo URL deduced from the Package URL (purl)."""
         return purl2url.get_repo_url(self.package_url)
 
+    def infer_download_url(self):
+        """Infer the download URL deduced from the Package URL (purl)."""
+        return download.infer_download_url(self.package_url)
+
     def get_url(self, name, params=None, include_identifier=False):
         if not params:
             params = [self.dataspace.name, quote_plus(str(self.uuid))]

product_portfolio/models.py

@@ -557,25 +557,40 @@ def assign_objects(self, related_objects, user, replace_version=False):
 
         return created_count, updated_count, unchanged_count
 
-    def scan_all_packages_task(self, user):
+    def scan_all_packages_task(self, user, infer_download_urls=False):
         """
         Submit a Scan request to ScanCode.io for each package assigned to this Product.
         Only packages with a proper download URL are sent.
         """
-        package_urls = [
+        if infer_download_urls:
+            self.improve_packages_from_purl()
+
+        package_download_urls = [
             package.download_url
-            for package in self.all_packages
+            for package in self.all_packages.has_download_url()
             if package.download_url.startswith(("http", "https"))
         ]
 
         tasks.scancodeio_submit_scan.delay(
-            uris=package_urls,
+            uris=package_download_urls,
             user_uuid=user.uuid,
             dataspace_uuid=user.dataspace.uuid,
         )
 
+    def improve_packages_from_purl(self):
+        """Infer missing packages download URL using the Package URL when possible."""
+        updated_packages = []
+
+        packages = self.all_packages.has_package_url().filter(models.Q(download_url=""))
+        for package in packages:
+            if download_url := package.infer_download_url():
+                package.update(download_url=download_url)
+                updated_packages.append(package)
+
+        return updated_packages
+
     def improve_packages_from_purldb(self, user):
-        """Update all Packages assigned to the Product using PurlDB data."""
+        """Update all packages assigned to thepProduct using PurlDB data."""
         updated_packages = []
         for package in self.packages.all():
             updated_fields = package.update_from_purldb(user)

product_portfolio/templates/product_portfolio/modals/scan_all_packages_modal.html

@@ -6,7 +6,7 @@ <h5 class="modal-title">Scan all Packages</h5>
         <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
       </div>
       <div class="modal-body bg-body-tertiary">
-        You are about to initiate multiple scans on the ScanCode.io server for all of the
+        You are about to initiate multiple scans on the ScanCode.io server for all the
         Packages assigned to your Product, either by direct assignment or by assignment to a
         Component assigned to your Product.<br><br>
         <strong>Note that this may take some time to complete.</strong><br><br>