Merge pull request #887 from aboutcode-org/create-federate-sbom-command

Create federate SBOM command

Author: JonoYang

docs/source/purldb/rest_api.rst

@@ -547,6 +547,28 @@ Using cURL to reindex a package:
         "status": "pkg:maven/org.elasticsearch/elasticsearch@7.17.9 has been queued for reindexing"
     }
 
+sbom
+^^^^^
+
+Generate a CycloneDX SBOM from this package instance.
+
+Using cURL to get an SBOM for a package:
+
+.. code-block:: console
+
+    api_url="https://public.purldb.io/api/packages/0bbdcf88-ad07-4970-9272-7d5f4c82cc7b/sbom/"
+    content_type="Content-Type: application/json"
+
+    curl -X GET "$api_url" -H "$content_type"
+
+.. code-block:: json
+
+    {
+        "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json",
+        "bomFormat": "CycloneDX",
+        "specVersion": "1.6",
+    }
+
 Filter by checksum
 ~~~~~~~~~~~~~~~~~~
 

minecode/management/commands/federate_packages.py

@@ -18,9 +18,7 @@
 from minecode_pipelines import pipes
 from packagedb import models as packagedb_models
 
-"""
-Utility command to find license oddities.
-"""
+
 logger = logging.getLogger(__name__)
 logging.basicConfig(stream=sys.stdout)
 logger.setLevel(logging.INFO)

minecode/management/commands/federate_sboms.py

@@ -0,0 +1,122 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import logging
+import sys
+from pathlib import Path
+
+from aboutcode.federated import DataFederation
+from commoncode import fileutils
+from minecode.management import federatedcode
+from minecode.management.commands import VerboseCommand
+from minecode_pipelines import pipes
+from packagedb import models as packagedb_models
+from packagedb import sbom
+
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(stream=sys.stdout)
+logger.setLevel(logging.INFO)
+
+TRACE = False
+if TRACE:
+    logger.setLevel(logging.DEBUG)
+
+
+PACKAGE_BATCH_SIZE = 1000
+
+
+def commit_message(commit_batch, total_commit_batch="many"):
+    from django.conf import settings
+
+    author_name = settings.FEDERATEDCODE_GIT_SERVICE_NAME
+    author_email = settings.FEDERATEDCODE_GIT_SERVICE_EMAIL
+    tool_name = "pkg:github/aboutcode-org/purldb"
+
+    return f"""\
+        Save CycloneDX SBOMs from PurlDB ({commit_batch}/{total_commit_batch})
+
+        Tool: {tool_name}@v{settings.PURLDB_VERSION}
+        Reference: https://{settings.ALLOWED_HOSTS[0]}
+
+        Signed-off-by: {author_name} <{author_email}>
+        """
+
+
+class Command(VerboseCommand):
+    help = "Save and commit CycloneDX SBOMs, generated from PackageDB package data. to FederatedCode repos."
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "-d",
+            "--working-directory",
+            type=str,
+            required=False,
+            help="Directory where FederatedCode repos will be cloned",
+        )
+
+    def handle(self, *args, **options):
+        logger.setLevel(self.get_verbosity(**options))
+        working_dir = options.get("working_directory")
+        if working_dir:
+            working_path = Path(working_dir)
+        else:
+            working_path = Path(fileutils.get_temp_dir())
+
+        # Clone data and config repo
+        data_federation = DataFederation.from_url(
+            name="aboutcode-data",
+            remote_root_url="https://github.com/aboutcode-data",
+        )
+        data_cluster = data_federation.get_cluster("cyclonedx16_sboms")
+
+        # TODO: do something more efficient
+        files_to_commit = []
+        commit_batch = 1
+        for i, package in enumerate(
+            packagedb_models.Package.objects.all().iterator(chunk_size=PACKAGE_BATCH_SIZE), start=1
+        ):
+            package_repo_name, datafile_path = data_cluster.get_datafile_repo_and_path(
+                purl=package.purl
+            )
+            _, package_repo = federatedcode.get_or_create_repository(
+                repo_name=package_repo_name,
+                working_path=working_path,
+                logger=logger.log,
+            )
+            package_sbom_data = sbom.to_cyclonedx(package)
+            sbom_file = pipes.write_package_data_to_file(
+                repo=package_repo,
+                relative_api_package_metadata_datafile_path=datafile_path,
+                package_data=package_sbom_data,
+            )
+            if sbom_file not in files_to_commit:
+                files_to_commit.append(sbom_file)
+
+            if len(files_to_commit) == PACKAGE_BATCH_SIZE:
+                federatedcode.commit_and_push_changes(
+                    commit_message=commit_message(commit_batch),
+                    repo=package_repo,
+                    files_to_commit=files_to_commit,
+                    logger=logger.log,
+                )
+                logger.log(f"Committed {i} SBOMs to {package_repo_name}")
+                files_to_commit.clear()
+                commit_batch += 1
+
+        if files_to_commit:
+            federatedcode.commit_and_push_changes(
+                commit_message=commit_message(commit_batch),
+                repo=package_repo,
+                files_to_commit=files_to_commit,
+                logger=logger.log,
+            )
+            logger.log(f"Committed {i} SBOMs to {package_repo_name}")
+            files_to_commit.clear()
+            commit_batch += 1

packagedb/sbom.py

@@ -34,7 +34,7 @@
 
 def get_cyclonedx_bom(package):
     """
-    Return a CycloneDX `Bom` object filled with provided `project` data.
+    Return a CycloneDX `Bom` object filled with data from `package`.
     See https://cyclonedx.org/use-cases/#dependency-graph
     """
 
@@ -76,9 +76,7 @@ def sort_bom_with_schema_ordering(bom_as_dict, schema_version):
 
 def to_cyclonedx(package, cyclonedx_version="1.6"):
     """
-    Generate output for the provided ``project`` in CycloneDX BOM format.
-    The output file is created in the ``project`` "output/" directory.
-    Return the path of the generated output file.
+    Return a CycloneDX SBOM of `package` as a Python dictionary.
     """
     schema_version = SchemaVersion.from_version(cyclonedx_version)