Refactor the dependency support in LoadSBOM #1145

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

scanpipe/pipelines/load_sbom.py

@@ -20,6 +20,7 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/nexB/scancode.io for support and download.
 
+from scanpipe.models import DiscoveredDependency
 from scanpipe.pipelines.scan_codebase import ScanCodebase
 from scanpipe.pipes import resolve
 
@@ -44,7 +45,7 @@ def steps(cls):
             cls.flag_empty_files,
             cls.flag_ignored_resources,
             cls.get_sbom_inputs,
-            cls.get_packages_from_sboms,
+            cls.get_data_from_sboms,
             cls.create_packages_from_sboms,
             cls.create_dependencies_from_sboms,
         )
@@ -53,13 +54,13 @@ def get_sbom_inputs(self):
         """Locate all the SBOMs among the codebase resources."""
         self.manifest_resources = resolve.get_manifest_resources(self.project)
 
-    def get_packages_from_sboms(self):
+    def get_data_from_sboms(self):
         """Get packages data from SBOMs."""
-        self.packages = resolve.get_packages(
+        self.packages, self.dependencies = resolve.get_data_from_manifests(
             project=self.project,
             package_registry=resolve.sbom_registry,
             manifest_resources=self.manifest_resources,
-            model="get_packages_from_sboms",
+            model="get_data_from_sboms",
         )
 
     def create_packages_from_sboms(self):
@@ -71,4 +72,11 @@ def create_packages_from_sboms(self):
 
     def create_dependencies_from_sboms(self):
         """Create the dependency relationship declared in the SBOMs."""
+        # TODO: Migrate the CycloneDX behavior too, see get_dependencies_from_manifest
         resolve.create_dependencies_from_packages_extra_data(project=self.project)
+
+        for dependency_data in self.dependencies:
+            DiscoveredDependency.create_from_data(
+                project=self.project,
+                dependency_data=dependency_data,
+            )

scanpipe/pipes/cyclonedx.py

@@ -155,12 +155,12 @@ def cyclonedx_component_to_package_data(cdx_component, dependencies=None):
     dependencies = dependencies or {}
     extra_data = {}
 
-    # Store the original bom_ref and dependencies for future processing.
     bom_ref = str(cdx_component.bom_ref)
-    if bom_ref:
-        extra_data["bom_ref"] = bom_ref
-        if depends_on := dependencies.get(bom_ref):
-            extra_data["depends_on"] = depends_on
+    if depends_on := dependencies.get(bom_ref):
+        extra_data["depends_on"] = depends_on
+
+    # Store the original "bom_ref" as package_uid for dependencies resolution.
+    package_uid = bom_ref
 
     package_url_dict = {}
     if cdx_component.purl:
@@ -176,6 +176,7 @@ def cyclonedx_component_to_package_data(cdx_component, dependencies=None):
         extra_data["nestedComponents"] = sorted(nested_purls)
 
     package_data = {
+        "package_uid": package_uid,
         "name": cdx_component.name,
         "extracted_license_statement": declared_license,
         "copyright": cdx_component.copyright,

scanpipe/pipes/resolve.py

@@ -57,12 +57,27 @@ def resolve_manifest_resources(resource, package_registry):
     return packages
 
 
-def get_packages(project, package_registry, manifest_resources, model=None):
+def get_dependencies_from_manifest(resource):
+    """Get dependency data from resource."""
+    dependencies = []
+
+    default_package_type = get_default_package_type(resource.location)
+    if not default_package_type:
+        return []
+
+    if default_package_type == "spdx":
+        dependencies = resolve_spdx_dependencies(input_location=resource.location)
+
+    return dependencies
+
+
+def get_data_from_manifests(project, package_registry, manifest_resources, model=None):
     """
-    Get package data from package manifests/lockfiles/SBOMs or
-    get package data for resolved packages from package requirements.
+    Get package and dependency data from package manifests/lockfiles/SBOMs or
+    for resolved packages from package requirements.
     """
     resolved_packages = []
+    resolved_dependencies = []
     sboms_headers = {}
 
     if not manifest_resources.exists():
@@ -73,7 +88,8 @@ def get_packages(project, package_registry, manifest_resources, model=None):
         return []
 
     for resource in manifest_resources:
-        if packages := resolve_manifest_resources(resource, package_registry):
+        packages = resolve_manifest_resources(resource, package_registry)
+        if packages:
             resolved_packages.extend(packages)
             if headers := get_manifest_headers(resource):
                 sboms_headers[resource.name] = headers
@@ -84,10 +100,14 @@ def get_packages(project, package_registry, manifest_resources, model=None):
                 resource=resource,
             )
 
+        dependencies = get_dependencies_from_manifest(resource)
+        if dependencies:
+            resolved_dependencies.extend(dependencies)
+
     if sboms_headers:
         project.update_extra_data({"sboms_headers": sboms_headers})
 
-    return resolved_packages
+    return resolved_packages, resolved_dependencies
 
 
 def create_packages_and_dependencies(project, packages, resolved=False):
@@ -136,7 +156,7 @@ def create_dependencies_from_packages_extra_data(project):
 
         for bom_ref in for_package.extra_data.get("depends_on", []):
             try:
-                resolved_to_package = project_packages.get(extra_data__bom_ref=bom_ref)
+                resolved_to_package = project_packages.get(package_uid=bom_ref)
             except (ObjectDoesNotExist, MultipleObjectsReturned):
                 project.add_error(
                     description=f"Could not find resolved_to package entry: {bom_ref}.",