Indexing for nixpkgs

minecode_pipelines/miners/nix.py

@@ -0,0 +1,143 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+
+import brotli
+import json
+import requests
+import time
+
+from packageurl import PackageURL
+
+
+NIX_TYPE = "nix"
+NIXPKGS = "nixpkgs"
+DELAY_MULTIPLIER = 2
+MAX_RETRIES = 3
+
+
+def get_all_nix_packages(channel_url, logger=None):
+    """Get all Nix packages"""
+    if logger:
+        logger(f"Fetching Nix packages from: {channel_url}")
+    response = requests.get(channel_url)
+    response.raise_for_status()
+
+    try:
+        decompressed_data = brotli.decompress(response.content)
+        data = json.loads(decompressed_data)
+    except brotli.error:
+        data = response.json()
+
+    packages_dict = data.get("packages", {})
+    return packages_dict
+
+
+def get_all_nix_packages_name(packages_dict, logger=None):
+    """Get all Nix packages name"""
+    all_package_names = []
+    packages = list(packages_dict.keys())
+    for attr_path in packages:
+        all_package_names.append(attr_path)
+
+    return {"packages": all_package_names}
+
+
+def load_nix_packages(packages_file):
+    with open(packages_file) as f:
+        packages_data = json.load(f)
+
+    return packages_data.get("packages", [])
+
+
+def get_nix_packageurls(name, packages_dict, logger=None):
+    packageurls = []
+    data = []
+    # Get all the version of a package from the following API:
+    # https://search.devbox.sh/v2/pkg?name={name}
+    url = f"https://search.devbox.sh/v2/pkg?name={name}"
+
+    for attempt in range(MAX_RETRIES):
+        try:
+            response = requests.get(url)
+            response.raise_for_status()
+            data = response.json()
+            break
+        except requests.HTTPError as e:
+            if response.status_code == 429:
+                retry_after = response.headers.get("Retry-After")
+                if retry_after:
+                    wait_time = int(retry_after)
+                else:
+                    wait_time = DELAY_MULTIPLIER**attempt
+                if logger:
+                    logger(f"Rate limited (429). Waiting {wait_time}s before retry...")
+                time.sleep(wait_time)
+                continue
+            elif response.status_code == 404:
+                # The package_dict contains the package's current version
+                # devbox.sh may have only index the top
+                # level, all others deeper level may not be indexed
+                # TODO: This can only get the current version.
+                version = get_version_from_package_dict(name, packages_dict)
+                if version:
+                    purl = PackageURL(type=NIX_TYPE, namespace=NIXPKGS, name=name, version=version)
+                else:
+                    purl = PackageURL(type=NIX_TYPE, namespace=NIXPKGS, name=name)
+                packageurls.append(purl.to_string())
+                return packageurls
+            else:
+                if logger:
+                    logger(f"Request failed: {e}")
+                    logger(f"HTTP error {response.status_code}: {response.text}")
+                return packageurls
+        except requests.RequestException as e:
+            if logger:
+                logger(f"Request failed: {e}")
+            return packageurls
+        except ValueError as e:
+            if logger:
+                logger(f"JSON parsing failed: {e}")
+            return packageurls
+    if not data:
+        # No data collected after retries
+        if logger:
+            logger(f"Max retries reached. Skipping package: {name}")
+        return packageurls
+
+    releases = data.get("releases", [])
+    for release in releases:
+        version = release.get("version")
+        if version:
+            purl = PackageURL(type=NIX_TYPE, namespace=NIXPKGS, name=name, version=version)
+            purl_string = purl.to_string()
+            if purl_string not in packageurls:
+                packageurls.append(purl_string)
+
+    return packageurls
+
+
+def get_version_from_package_dict(name, package_dict):
+    """Get the version data from the package_dict"""
+    package_info = package_dict.get(name)
+    if not package_info:
+        return None
+    return package_info.get("version")
+
+
+def yield_nix_package_data(name, packageurls=[]):
+    for purl in packageurls:
+        package_url = PackageURL.from_string(purl)
+        package_data_url = (
+            f"https://search.devbox.sh/v2/resolve?name={name}&version={package_url.version}"
+        )
+        response = requests.get(package_data_url)
+        if not response.ok:
+            continue
+        yield purl, response.json()

minecode_pipelines/pipelines/mine_nix.py

@@ -0,0 +1,107 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+
+from minecode_pipelines.pipes import nix
+from minecode_pipelines.pipelines import MineCodeBasePipeline
+from minecode_pipelines.pipelines import _mine_and_publish_packageurls
+
+
+class MineNix(MineCodeBasePipeline):
+    """
+    Mine PackageURLs from NixOS-Packages and publish them to FederatedCode.
+    """
+
+    package_batch_size = 5
+    nixpkgs_repo_url = "https://github.com/NixOS/nixpkgs"
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.check_federatedcode_eligibility,
+            cls.create_federatedcode_working_dir,
+            cls.mine_nix_packages,
+            cls.get_nixpkgs_packages_to_sync,
+            cls.fetch_federation_config,
+            cls.mine_and_publish_packageurls,
+            cls.update_state_and_checkpoints,
+            cls.delete_working_dir,
+        )
+
+    def mine_nix_packages(self):
+        """Mine Nix package names from NixOS packages or checkpoint."""
+        (self.nix_packages_dict, self.nix_packages, self.state, self.config_repo) = (
+            nix.mine_nix_packages(logger=self.log)
+        )
+
+    def get_nixpkgs_packages_to_sync(self):
+        """Get Nixpkgs packages which needs to be synced using checkpoint."""
+        self.packages, self.synced_packages = nix.get_nix_packages_to_sync(
+            packages_file=self.nix_packages,
+            state=self.state,
+            logger=self.log,
+        )
+
+    def packages_count(self):
+        return len(self.packages)
+
+    def mine_packageurls(self):
+        """Yield Nix packageURLs for all mined Nix package names."""
+        self.packages_mined = []
+        yield from nix.mine_and_publish_nix_packageurls(
+            packages_dict=self.nix_packages_dict,
+            packages_to_sync=self.packages,
+            packages_mined=self.packages_mined,
+            logger=self.log,
+        )
+
+    def save_check_point(self):
+        nix.save_mined_packages_in_checkpoint(
+            packages_mined=self.packages_mined,
+            synced_packages=self.synced_packages,
+            config_repo=self.config_repo,
+            logger=self.log,
+        )
+        self.packages_mined = []
+
+    def mine_and_publish_packageurls(self):
+        """Mine and publish PackageURLs."""
+        _mine_and_publish_packageurls(
+            packageurls=self.mine_packageurls(),
+            total_package_count=self.packages_count(),
+            data_clusters=self.data_clusters,
+            checked_out_repos=self.checked_out_repos,
+            working_path=self.working_path,
+            append_purls=self.append_purls,
+            commit_msg_func=self.commit_message,
+            logger=self.log,
+            checkpoint_func=self.save_check_point,
+            checkpoint_on_commit=True,
+            batch_size=self.package_batch_size,
+        )
+
+    def update_state_and_checkpoints(self):
+        nix.update_state_and_checkpoints(
+            state=self.state,
+            config_repo=self.config_repo,
+            logger=self.log,
+        )

minecode_pipelines/pipes/__init__.py

@@ -92,6 +92,8 @@ def update_checkpoints_file_in_github(checkpoints_file, cloned_repo, path):
     from scanpipe.pipes.federatedcode import commit_and_push_changes
 
     checkpoint_path = os.path.join(cloned_repo.working_dir, path)
+    # Make sure the directory exists
+    os.makedirs(os.path.dirname(checkpoint_path), exist_ok=True)
     shutil.move(checkpoints_file, checkpoint_path)
     commit_message = """Update federatedcode purl mining checkpoint"""
     commit_and_push_changes(