diff --git a/minecode_pipelines/miners/nix.py b/minecode_pipelines/miners/nix.py new file mode 100644 index 00000000..6a3b77d8 --- /dev/null +++ b/minecode_pipelines/miners/nix.py @@ -0,0 +1,143 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + + +import brotli +import json +import requests +import time + +from packageurl import PackageURL + + +NIX_TYPE = "nix" +NIXPKGS = "nixpkgs" +DELAY_MULTIPLIER = 2 +MAX_RETRIES = 3 + + +def get_all_nix_packages(channel_url, logger=None): + """Get all Nix packages""" + if logger: + logger(f"Fetching Nix packages from: {channel_url}") + response = requests.get(channel_url) + response.raise_for_status() + + try: + decompressed_data = brotli.decompress(response.content) + data = json.loads(decompressed_data) + except brotli.error: + data = response.json() + + packages_dict = data.get("packages", {}) + return packages_dict + + +def get_all_nix_packages_name(packages_dict, logger=None): + """Get all Nix packages name""" + all_package_names = [] + packages = list(packages_dict.keys()) + for attr_path in packages: + all_package_names.append(attr_path) + + return {"packages": all_package_names} + + +def load_nix_packages(packages_file): + with open(packages_file) as f: + packages_data = json.load(f) + + return packages_data.get("packages", []) + + +def get_nix_packageurls(name, packages_dict, logger=None): + packageurls = [] + data = [] + # Get all the version of a package from the following API: + # https://search.devbox.sh/v2/pkg?name={name} + url = f"https://search.devbox.sh/v2/pkg?name={name}" + + for attempt in range(MAX_RETRIES): + try: + response = requests.get(url) + response.raise_for_status() + data = response.json() + break + except requests.HTTPError as e: + if response.status_code == 429: + retry_after = response.headers.get("Retry-After") + if retry_after: + wait_time = int(retry_after) + else: + wait_time = DELAY_MULTIPLIER**attempt + if logger: + logger(f"Rate limited (429). Waiting {wait_time}s before retry...") + time.sleep(wait_time) + continue + elif response.status_code == 404: + # The package_dict contains the package's current version + # devbox.sh may have only index the top + # level, all others deeper level may not be indexed + # TODO: This can only get the current version. + version = get_version_from_package_dict(name, packages_dict) + if version: + purl = PackageURL(type=NIX_TYPE, namespace=NIXPKGS, name=name, version=version) + else: + purl = PackageURL(type=NIX_TYPE, namespace=NIXPKGS, name=name) + packageurls.append(purl.to_string()) + return packageurls + else: + if logger: + logger(f"Request failed: {e}") + logger(f"HTTP error {response.status_code}: {response.text}") + return packageurls + except requests.RequestException as e: + if logger: + logger(f"Request failed: {e}") + return packageurls + except ValueError as e: + if logger: + logger(f"JSON parsing failed: {e}") + return packageurls + if not data: + # No data collected after retries + if logger: + logger(f"Max retries reached. Skipping package: {name}") + return packageurls + + releases = data.get("releases", []) + for release in releases: + version = release.get("version") + if version: + purl = PackageURL(type=NIX_TYPE, namespace=NIXPKGS, name=name, version=version) + purl_string = purl.to_string() + if purl_string not in packageurls: + packageurls.append(purl_string) + + return packageurls + + +def get_version_from_package_dict(name, package_dict): + """Get the version data from the package_dict""" + package_info = package_dict.get(name) + if not package_info: + return None + return package_info.get("version") + + +def yield_nix_package_data(name, packageurls=[]): + for purl in packageurls: + package_url = PackageURL.from_string(purl) + package_data_url = ( + f"https://search.devbox.sh/v2/resolve?name={name}&version={package_url.version}" + ) + response = requests.get(package_data_url) + if not response.ok: + continue + yield purl, response.json() diff --git a/minecode_pipelines/pipelines/mine_nix.py b/minecode_pipelines/pipelines/mine_nix.py new file mode 100644 index 00000000..4ebe5930 --- /dev/null +++ b/minecode_pipelines/pipelines/mine_nix.py @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + + +from minecode_pipelines.pipes import nix +from minecode_pipelines.pipelines import MineCodeBasePipeline +from minecode_pipelines.pipelines import _mine_and_publish_packageurls + + +class MineNix(MineCodeBasePipeline): + """ + Mine PackageURLs from NixOS-Packages and publish them to FederatedCode. + """ + + package_batch_size = 5 + nixpkgs_repo_url = "https://github.com/NixOS/nixpkgs" + + @classmethod + def steps(cls): + return ( + cls.check_federatedcode_eligibility, + cls.create_federatedcode_working_dir, + cls.mine_nix_packages, + cls.get_nixpkgs_packages_to_sync, + cls.fetch_federation_config, + cls.mine_and_publish_packageurls, + cls.update_state_and_checkpoints, + cls.delete_working_dir, + ) + + def mine_nix_packages(self): + """Mine Nix package names from NixOS packages or checkpoint.""" + (self.nix_packages_dict, self.nix_packages, self.state, self.config_repo) = ( + nix.mine_nix_packages(logger=self.log) + ) + + def get_nixpkgs_packages_to_sync(self): + """Get Nixpkgs packages which needs to be synced using checkpoint.""" + self.packages, self.synced_packages = nix.get_nix_packages_to_sync( + packages_file=self.nix_packages, + state=self.state, + logger=self.log, + ) + + def packages_count(self): + return len(self.packages) + + def mine_packageurls(self): + """Yield Nix packageURLs for all mined Nix package names.""" + self.packages_mined = [] + yield from nix.mine_and_publish_nix_packageurls( + packages_dict=self.nix_packages_dict, + packages_to_sync=self.packages, + packages_mined=self.packages_mined, + logger=self.log, + ) + + def save_check_point(self): + nix.save_mined_packages_in_checkpoint( + packages_mined=self.packages_mined, + synced_packages=self.synced_packages, + config_repo=self.config_repo, + logger=self.log, + ) + self.packages_mined = [] + + def mine_and_publish_packageurls(self): + """Mine and publish PackageURLs.""" + _mine_and_publish_packageurls( + packageurls=self.mine_packageurls(), + total_package_count=self.packages_count(), + data_clusters=self.data_clusters, + checked_out_repos=self.checked_out_repos, + working_path=self.working_path, + append_purls=self.append_purls, + commit_msg_func=self.commit_message, + logger=self.log, + checkpoint_func=self.save_check_point, + checkpoint_on_commit=True, + batch_size=self.package_batch_size, + ) + + def update_state_and_checkpoints(self): + nix.update_state_and_checkpoints( + state=self.state, + config_repo=self.config_repo, + logger=self.log, + ) diff --git a/minecode_pipelines/pipes/__init__.py b/minecode_pipelines/pipes/__init__.py index 9e69c180..4317f6a9 100644 --- a/minecode_pipelines/pipes/__init__.py +++ b/minecode_pipelines/pipes/__init__.py @@ -92,6 +92,8 @@ def update_checkpoints_file_in_github(checkpoints_file, cloned_repo, path): from scanpipe.pipes.federatedcode import commit_and_push_changes checkpoint_path = os.path.join(cloned_repo.working_dir, path) + # Make sure the directory exists + os.makedirs(os.path.dirname(checkpoint_path), exist_ok=True) shutil.move(checkpoints_file, checkpoint_path) commit_message = """Update federatedcode purl mining checkpoint""" commit_and_push_changes( diff --git a/minecode_pipelines/pipes/nix.py b/minecode_pipelines/pipes/nix.py new file mode 100644 index 00000000..90d187be --- /dev/null +++ b/minecode_pipelines/pipes/nix.py @@ -0,0 +1,273 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +from datetime import datetime + + +from minecode_pipelines.pipes import fetch_checkpoint_from_github +from minecode_pipelines.pipes import update_checkpoints_in_github +from minecode_pipelines.pipes import update_checkpoints_file_in_github +from minecode_pipelines.pipes import get_mined_packages_from_checkpoint +from minecode_pipelines.pipes import update_mined_packages_in_checkpoint +from minecode_pipelines.pipes import update_checkpoint_state +from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO +from minecode_pipelines.pipes import INITIAL_SYNC_STATE +from minecode_pipelines.pipes import PERIODIC_SYNC_STATE +from minecode_pipelines.pipes import write_packages_json +from minecode_pipelines.pipes import compress_packages_file +from minecode_pipelines.pipes import decompress_packages_file +from minecode_pipelines.pipes import fetch_checkpoint_by_git + +from minecode_pipelines.miners.nix import get_all_nix_packages +from minecode_pipelines.miners.nix import get_all_nix_packages_name +from minecode_pipelines.miners.nix import load_nix_packages +from minecode_pipelines.miners.nix import get_nix_packageurls +from minecode_pipelines.miners.nix import yield_nix_package_data +from minecode_pipelines.miners.nix import NIX_TYPE + + +from minecode_pipelines.utils import get_temp_dir + +from packageurl import PackageURL + +from scanpipe.pipes.federatedcode import clone_repository +from scanpipe.pipes.federatedcode import delete_local_clone + + +PACKAGE_FILE_NAME = "NixPackages.json" +COMPRESSED_PACKAGE_FILE_NAME = "NixPackages.json.gz" +NIX_REPLICATE_CHECKPOINT_PATH = "nix/" + PACKAGE_FILE_NAME +COMPRESSED_NIX_REPLICATE_CHECKPOINT_PATH = "nix/" + COMPRESSED_PACKAGE_FILE_NAME +NIX_CHECKPOINT_PATH = "nix/checkpoints.json" +NIX_PACKAGES_CHECKPOINT_PATH = "nix/packages_checkpoint.json" +PACKAGE_BATCH_SIZE = 700 + +CHANNEL_URL = "https://channels.nixos.org/nixos-unstable/packages.json.br" + + +def mine_nix_packages(logger=None): + """ + Mine nix package names from nixos.org and save to checkpoints, + or get packages from saved checkpoints. We have 3 cases: + + 1. first sync: we get latest set of packages from nixos.org and save the + timestamp to checkpoints. + 2. initial sync: we get packages from checkpoint which we're trying to + sync up to date + 3. periodic sync: pull in newly released packages from nixos.org. + """ + nix_checkpoints = fetch_checkpoint_from_github( + config_repo=MINECODE_PIPELINES_CONFIG_REPO, + checkpoint_path=NIX_CHECKPOINT_PATH, + ) + state = nix_checkpoints.get("state") + if logger: + logger(f"Mining state from checkpoint: {state}") + + config_repo = clone_repository( + repo_url=MINECODE_PIPELINES_CONFIG_REPO, + clone_path=get_temp_dir(), + logger=logger, + ) + + packages_dict = {} + + if not state: + packages_dict = get_all_nix_packages(CHANNEL_URL, logger=logger) + packages = get_all_nix_packages_name(packages_dict, logger=logger) + packages_file = write_packages_json( + packages=packages, + name=PACKAGE_FILE_NAME, + ) + + compressed_packages_file = packages_file + ".gz" + compress_packages_file( + packages_file=packages_file, + compressed_packages_file=compressed_packages_file, + ) + + update_checkpoints_file_in_github( + checkpoints_file=compressed_packages_file, + cloned_repo=config_repo, + path=COMPRESSED_NIX_REPLICATE_CHECKPOINT_PATH, + ) + + if logger: + logger(f"Updating checkpoint mining state to: {INITIAL_SYNC_STATE}") + + update_nix_checkpoints( + cloned_repo=config_repo, + state=INITIAL_SYNC_STATE, + checkpoint_path=NIX_CHECKPOINT_PATH, + logger=logger, + ) + + elif state == INITIAL_SYNC_STATE: + if logger: + logger("Getting packages to sync from nix checkpoint") + + compressed_packages_file = fetch_checkpoint_by_git( + cloned_repo=config_repo, + checkpoint_path=COMPRESSED_NIX_REPLICATE_CHECKPOINT_PATH, + ) + packages_file = decompress_packages_file( + compressed_packages_file=compressed_packages_file, + name=PACKAGE_FILE_NAME, + ) + + elif state == PERIODIC_SYNC_STATE: + packages_dict = get_all_nix_packages(CHANNEL_URL, logger=logger) + packages = get_all_nix_packages_name(packages_dict, logger=logger) + packages_file = write_packages_json( + packages=packages, + name=PACKAGE_FILE_NAME, + ) + + return packages_dict, packages_file, state, config_repo + + +def update_nix_checkpoints( + cloned_repo, + checkpoint_path, + state=None, + config_repo=MINECODE_PIPELINES_CONFIG_REPO, + logger=None, +): + checkpoint = fetch_checkpoint_from_github( + config_repo=config_repo, + checkpoint_path=checkpoint_path, + ) + if state: + checkpoint["state"] = state + + checkpoint["date"] = str(datetime.now()) + update_checkpoints_in_github( + checkpoint=checkpoint, + cloned_repo=cloned_repo, + path=checkpoint_path, + logger=logger, + ) + + +def get_nix_packages_to_sync(packages_file, state, logger=None): + if logger: + logger(f"Mining state: {state}") + + packages = load_nix_packages(packages_file) + if logger: + logger(f"# of package names fetched from index/checkpoint: {len(packages)}") + + if not packages: + return + + if not state: + packages_to_sync = packages + if logger: + logger(f"Starting package mining for {len(packages_to_sync)} packages") + + synced_packages = [] + + elif state == INITIAL_SYNC_STATE or state == PERIODIC_SYNC_STATE: + synced_packages = get_mined_packages_from_checkpoint( + config_repo=MINECODE_PIPELINES_CONFIG_REPO, + checkpoint_path=NIX_PACKAGES_CHECKPOINT_PATH, + ) + if state == INITIAL_SYNC_STATE: + packages_to_sync = synced_packages + else: + packages_to_sync = list(set(packages).difference(set(synced_packages))) + if logger: + logger( + f"Starting initial package mining for {len(packages_to_sync)} packages from checkpoint" + ) + + return packages_to_sync, synced_packages + + +def mine_and_publish_nix_packageurls(packages_dict, packages_to_sync, packages_mined, logger=None): + if logger: + logger("Starting package mining for a batch of packages") + + for package_name in packages_to_sync: + # fetch packageURLs for package + if logger: + logger(f"getting packageURLs for package: {package_name}") + + packageurls = get_nix_packageurls(package_name, packages_dict, logger=logger) + if not packageurls: + if logger: + logger(f"Could not fetch package versions for package: {package_name}") + continue + + purls_and_package_data = yield_nix_package_data(package_name, packageurls) + # We currently only work with the official nixpkgs repository, so + # we can set the namespace to "nixpkgs" + base_purl = PackageURL(type=NIX_TYPE, namespace="nixpkgs", name=package_name).to_string() + packages_mined.append(package_name) + if purls_and_package_data: + yield base_purl, packageurls, purls_and_package_data + else: + yield base_purl, packageurls, [] + + +def save_mined_packages_in_checkpoint(packages_mined, synced_packages, config_repo, logger=None): + # As we are mining the packages to sync with the index, + # we need to update mined packages checkpoint for every batch + # so we can continue mining the other packages after restarting + if logger: + logger(f"Checkpointing processed packages to: {NIX_PACKAGES_CHECKPOINT_PATH}") + + packages_checkpoint = packages_mined + synced_packages + update_mined_packages_in_checkpoint( + packages=packages_checkpoint, + config_repo=MINECODE_PIPELINES_CONFIG_REPO, + cloned_repo=config_repo, + checkpoint_path=NIX_PACKAGES_CHECKPOINT_PATH, + logger=logger, + ) + + +def update_state_and_checkpoints(state, config_repo, logger=None): + # If we are finished mining all the packages in the initial sync, we can now + # periodically sync the packages from latest + if state == INITIAL_SYNC_STATE: + if logger: + logger(f"{INITIAL_SYNC_STATE} completed. starting: {PERIODIC_SYNC_STATE}") + update_checkpoint_state( + cloned_repo=config_repo, + state=PERIODIC_SYNC_STATE, + checkpoint_path=NIX_CHECKPOINT_PATH, + ) + + # If we are finished mining all the packages in the periodic sync, we can now update + # the last sequence updated + if state == PERIODIC_SYNC_STATE: + update_nix_checkpoints( + cloned_repo=config_repo, + checkpoint_path=NIX_CHECKPOINT_PATH, + state=PERIODIC_SYNC_STATE, + logger=logger, + ) + + if logger: + logger(f"Deleting local clone at: {config_repo.working_dir}") + delete_local_clone(config_repo) diff --git a/pyproject-minecode_pipelines.toml b/pyproject-minecode_pipelines.toml index 076fe081..7eec0686 100644 --- a/pyproject-minecode_pipelines.toml +++ b/pyproject-minecode_pipelines.toml @@ -44,7 +44,8 @@ dependencies = [ "ftputil >= 5.1.0", "jawa >= 2.2.0", "arrow >= 1.3.0", - "beautifulsoup4 >= 4.13.4" + "beautifulsoup4 >= 4.13.4", + "brotli>=1.2.0" ] urls = { Homepage = "https://github.com/aboutcode-org/purldb" } @@ -62,6 +63,7 @@ mine_cpan = "minecode_pipelines.pipelines.mine_cpan:MineCpan" mine_cran = "minecode_pipelines.pipelines.mine_cran:MineCran" mine_swift = "minecode_pipelines.pipelines.mine_swift:MineSwift" mine_composer = "minecode_pipelines.pipelines.mine_composer:MineComposer" +mine_nix = "minecode_pipelines.pipelines.mine_nix:MineNix" mine_crates = "minecode_pipelines.pipelines.mine_crates:MineCrates" [tool.bumpversion] diff --git a/requirements.txt b/requirements.txt index 57c86ef4..adda81a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ binaryornot==0.4.4 bitarray==3.8.0 bleach==6.2.0 boolean.py==5.0 +brotli==1.2.0 certifi==2025.11.12 cffi==2.0.0 chardet==5.2.0 diff --git a/setup.cfg b/setup.cfg index 827ff26c..20e118ad 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,6 +42,7 @@ python_requires = >=3.10 install_requires = aboutcode.pipeline >= 0.2.1 arrow >= 1.3.0 + brotli>=1.2.0 debian-inspector >= 31.1.1 commoncode >= 32.3.0 Django >= 5.1.11