diff --git a/minecode/collectors/nix.py b/minecode/collectors/nix.py new file mode 100644 index 00000000..03a39607 --- /dev/null +++ b/minecode/collectors/nix.py @@ -0,0 +1,182 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging +import shutil +import subprocess +import json +from packageurl import PackageURL +from minecode.miners.nix import build_packages, verify_url_existence +from fetchcode import fetch_json_response + +from minecode import priority_router +from packagedb.models import PackageContentType + +logger = logging.getLogger(__name__) +handler = logging.StreamHandler() +logger.addHandler(handler) +logger.setLevel(logging.INFO) + + +def map_nix_package(package_url, pipelines, priority=0): + """ + Add a composer `package_url` to the PackageDB. + """ + from minecode.model_utils import add_package_to_scan_queue, merge_or_create_package + + # Check if the `nix` command is available on the system + have_nix = False + if shutil.which("nix") is not None: + have_nix = True + + namespace = package_url.namespace + # We will only work with the official nixpkgs repository. + # It is impossible to handle all third‑party or custom repositories. + if not namespace.lower() == "nixpkgs": + return None + + data = get_nix_package_data(package_url) + + if not data and have_nix: + data = get_nix_package_data_via_cli(package_url) + clean_garbage() + + if data: + packages = build_packages(data, package_url) + + error = None + for package in packages: + package.extra_data["package_content"] = PackageContentType.BINARY + db_package, _, _, error = merge_or_create_package(package, visit_level=0) + if error: + break + + if db_package: + add_package_to_scan_queue( + package=db_package, pipelines=pipelines, priority=priority + ) + + return error + else: + return f"Failed to fetch package data for {package_url}" + + +def get_nix_package_data(purl): + """ + Fetch package data from https://search.devbox.sh/. + """ + + api_url = f"https://search.devbox.sh/v2/pkg?name={purl.name}" + if not verify_url_existence(api_url): + return None + + return fetch_json_response(api_url) + + +def parse_license(license_data): + """ + Parse a license object and return a string representation of the license. + """ + return ( + license_data.get("spdxId") + or license_data.get("fullName") + or license_data.get("shortName") + or str(license_data) + ) + + +def get_nix_package_data_via_cli(package_url): + """ + Fetch package metadata using the Nix CLI (mainly for packages that + Devbox doesn't index). + """ + # Create a Nix expression to pull just the fields we want. + package_name = package_url.name + package_version = package_url.version + + nix_exp = f""" + let + pkgs = import {{}}; + pkg = pkgs.{package_name}; + outputNames = if pkg ? outputs then pkg.outputs else ["out"]; + in {{ + name = pkg.pname or pkg.name or ""; + version = pkg.version or ""; + storePath = pkg.outPath or ""; + outputs = builtins.listToAttrs (map (name: {{ name = name; value = pkg.${{name}}.outPath; }}) outputNames); + description = pkg.meta.description or ""; + homepage = pkg.meta.homepage or ""; + license = pkg.meta.license or {{}}; + system = pkg.system or ""; + }} + """ + + cmd = ["nix-instantiate", "--eval", "--json", "--strict", "-E", nix_exp] + + try: + result = subprocess.run(cmd, capture_output=True, text=True, check=True) # noqa: S603 + pkg_data = json.loads(result.stdout) + + version = pkg_data.get("version", "") + if package_version: + if not version or version != package_version: + print(f"Cannot find version: {package_version}, got {version}") + return None + + output_list = [ + {"name": pname, "path": ppath} for pname, ppath in pkg_data.get("outputs", {}).items() + ] + + # Return a similar JSON layout as Devbox + return { + "summary": pkg_data.get("description", ""), + "homepage_url": pkg_data.get("homepage", ""), + "license": parse_license(pkg_data.get("license", {})), + "releases": [ + { + "version": version, + "platforms": [{"system": pkg_data.get("system", ""), "outputs": output_list}], + } + ], + } + except Exception as e: + print( + f"Failed to fetch Nix package data for {package_url}: {e.stderr if hasattr(e, 'stderr') else e}" + ) + return None + + +def clean_garbage(): + """ + Delete all unreferenced downloaded tarballs and evaluation caches + """ + nix_store = shutil.which("nix-store") + if nix_store: + subprocess.run([nix_store, "--gc"], capture_output=True) # noqa: S603 + else: + raise FileNotFoundError("nix-store not found in PATH") + + +@priority_router.route("pkg:nix/nixpkgs/.*") +def process_request(purl_str, **kwargs): + """ + Process `priority_resource_uri` containing a nix Package URL (PURL). + """ + from minecode.model_utils import DEFAULT_PIPELINES + + addon_pipelines = kwargs.get("addon_pipelines", []) + pipelines = DEFAULT_PIPELINES + tuple(addon_pipelines) + priority = kwargs.get("priority", 0) + + package_url = PackageURL.from_string(purl_str) + + error_msg = map_nix_package(package_url, pipelines, priority) + + if error_msg: + return error_msg diff --git a/minecode/miners/nix.py b/minecode/miners/nix.py new file mode 100644 index 00000000..57b7610a --- /dev/null +++ b/minecode/miners/nix.py @@ -0,0 +1,127 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from packagedcode import models as scan_models +from packageurl import PackageURL + +import requests + + +def get_nix_download_url(path): + """ + Construct a download url from cache.nixos.org based on the /nix/store/ + path + """ + narinfo_hash = path.replace("/nix/store/", "").split("-")[0] + narinfo_url = f"https://cache.nixos.org/{narinfo_hash}.narinfo" + url_path = get_narinfo_url(narinfo_url) + return f"https://cache.nixos.org/{url_path}" + + +def get_narinfo_url(narinfo_url): + """ + Visit the narinfo url, parsed and return the URL value + """ + # Fetch the narinfo file + response = requests.get(narinfo_url) + response.raise_for_status() + + # Parse line by line + for line in response.text.splitlines(): + if line.startswith("URL:"): + # Strip off "URL:" and any whitespace + return line.split(":", 1)[1].strip() + return None + + +def build_packages(metadata_dict, purl): + package_version = purl.version + + description = metadata_dict.get("summary", "") + homepage_url = metadata_dict.get("homepage_url", "") + license = metadata_dict.get("license", []) + extracted_license_statement = license if isinstance(license, list) else [license] + + releases = metadata_dict.get("releases", []) + for release in releases: + version = release.get("version", "") + if package_version and version != package_version: + continue + common_data = dict( + name=purl.name, + namespace=purl.namespace, + version=version, + description=description, + homepage_url=homepage_url, + extracted_license_statement=extracted_license_statement, + ) + + platforms = release.get("platforms", "") + for platform in platforms: + date = platform.get("date") or None + platform_system = platform.get("system", "") + commit = platform.get("commit_hash", "") + platform_outputs = platform.get("outputs") or None + + if not platform_outputs: + continue + for platform_output in platform_outputs: + output_name = platform_output.get("name") + path = platform_output.get("path") + narinfo_url = path.replace("/nix/store/", "").split("-")[0] + download_url = get_nix_download_url(narinfo_url) + download_data = dict( + datasource_id="nix_pkginfo", + type="nix", + download_url=download_url, + release_date=date, + ) + download_data.update(common_data) + package = scan_models.PackageData.from_data(download_data) + package.datasource_id = "nix_api_metadata" + qualifiers = {} + if platform_system: + qualifiers["system"] = platform_system + if commit: + qualifiers["commit"] = commit + if output_name: + qualifiers["output"] = output_name + updated_purl = update_purl_with_version_qualifiers(purl, version, qualifiers) + package.set_purl(updated_purl) + yield package + + +def update_purl_with_version_qualifiers(purl, version, qualifiers): + """ + Update a PackageURL with the given version and qualifiers. + """ + return PackageURL( + type=purl.type, + namespace=purl.namespace, + name=purl.name, + version=version, + qualifiers=qualifiers, + subpath=purl.subpath, + ) + + +def verify_url_existence(url): + """ + Perform a fast HTTP HEAD request to check if a generated URL is valid. + """ + try: + response = requests.head(url, allow_redirects=True, timeout=5) + if response.status_code == 200: + return True + elif response.status_code in (403, 429, 409): # forbidden, rate limit, conflict + return True # resource exists but not accessible + else: + return False + except Exception: + return False diff --git a/minecode/tests/collectors/test_nix.py b/minecode/tests/collectors/test_nix.py new file mode 100644 index 00000000..0923d48b --- /dev/null +++ b/minecode/tests/collectors/test_nix.py @@ -0,0 +1,144 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import json +import os + +from django.test import TestCase as DjangoTestCase + +from packageurl import PackageURL + +import packagedb +from minecode.collectors import nix +from minecode.tests import FIXTURES_REGEN +from minecode.utils_test import JsonBasedTesting + + +class NixPriorityQueueTests(JsonBasedTesting, DjangoTestCase): + test_data_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "testfiles") + + def setUp(self): + super().setUp() + self.expected_json_loc = self.get_test_loc("nix/SDL_mixer_package-expected.json") + with open(self.expected_json_loc) as f: + self.expected_json_contents = json.load(f) + """ + self.scan_package = NpmPackageJsonHandler._parse( + json_data=self.expected_json_contents, + ) + """ + + def test_get_nix_package_data(self, regen=FIXTURES_REGEN): + purl = PackageURL.from_string("pkg:nix/nixpkgs/SDL_mixer@1.2.12") + json_contents = nix.get_nix_package_data(purl) + if regen: + with open(self.expected_json_loc, "w") as f: + json.dump(json_contents, f, indent=3, separators=(",", ":")) + self.assertEqual(self.expected_json_contents, json_contents) + + def test_parse_license(self): + license_data = { + "deprecated": "false", + "free": "true", + "fullName": 'BSD 3-clause "New" or "Revised" License', + "licenseType": "simple", + "redistributable": "true", + "shortName": "bsd3", + "spdxId": "BSD-3-Clause", + "url": "https://spdx.org/licenses/BSD-3-Clause.html", + } + result = nix.parse_license(license_data) + self.assertEqual("BSD-3-Clause", result) + + def test_get_nix_package_data_via_cli(self): + import shutil + + if shutil.which("nix") is not None: + metadata = { + "summary": "Scientific tools for Python", + "homepage_url": "https://numpy.org/", + "license": "BSD-3-Clause", + "releases": [ + { + "version": "2.4.4", + "platforms": [ + { + "system": "x86_64-linux", + "outputs": [ + { + "name": "dist", + "path": "/nix/store/3wmy167jrryy19h6i6hnfbzy4j0ndkma-python3.13-numpy-2.4.4-dist", + }, + { + "name": "out", + "path": "/nix/store/l59n6vzkswz23y6s4pr6cmv2p4dpd5f0-python3.13-numpy-2.4.4", + }, + ], + } + ], + } + ], + } + result = nix.get_nix_package_data_via_cli( + PackageURL.from_string("pkg:nix/nixpkgs/python3Packages.numpy@2.4.4") + ) + self.assertEqual(metadata, result) + + def test_map_nix_package(self): + package_count = packagedb.models.Package.objects.all().count() + self.assertEqual(0, package_count) + package_url = PackageURL.from_string("pkg:nix/nixpkgs/SDL_mixer@1.2.12") + nix.map_nix_package(package_url, ("test_pipeline")) + package_count = packagedb.models.Package.objects.all().count() + # There are 4 different systems and each system has 2 outputs, so + # we expect 8 packages to be created in total. + self.assertEqual(8, package_count) + # package = packagedb.models.Package.objects.all().first() + packages = packagedb.models.Package.objects.all() + + expected_purl_data = [ + ( + "pkg:nix/nixpkgs/SDL_mixer@1.2.12?commit=3d46470bb3030020f7e1361f33514854f5bfa86d&output=out&system=aarch64-linux", + "https://cache.nixos.org/nar/07q6kl7ndvxi550gk7wm8j7m3lhbfbl5pshgx0amx38p4pq4haml.nar.zst", + ), + ( + "pkg:nix/nixpkgs/SDL_mixer@1.2.12?commit=3d46470bb3030020f7e1361f33514854f5bfa86d&output=dev&system=aarch64-linux", + "https://cache.nixos.org/nar/13vh3dfwp5bcy75csf6l69zyxa8y9w0azy8drx9nacdqq84jzhl1.nar.zst", + ), + ( + "pkg:nix/nixpkgs/SDL_mixer@1.2.12?commit=3d46470bb3030020f7e1361f33514854f5bfa86d&output=out&system=aarch64-darwin", + "https://cache.nixos.org/nar/10iscj2cnh5yhdgc5rnb9rmzr5galwmzmar495c9k410k7afc2kw.nar.zst", + ), + ( + "pkg:nix/nixpkgs/SDL_mixer@1.2.12?commit=3d46470bb3030020f7e1361f33514854f5bfa86d&output=dev&system=aarch64-darwin", + "https://cache.nixos.org/nar/11zwp2lzw481agxqgg7f8ikj4zbivl7a2r0nvwsr107c963f59pl.nar.zst", + ), + ( + "pkg:nix/nixpkgs/SDL_mixer@1.2.12?commit=3d46470bb3030020f7e1361f33514854f5bfa86d&output=out&system=x86_64-darwin", + "https://cache.nixos.org/nar/1y793cshy2hvdch0g3svi8bg0jlnx96jxsyi7960c1272cq7ricf.nar.zst", + ), + ( + "pkg:nix/nixpkgs/SDL_mixer@1.2.12?commit=3d46470bb3030020f7e1361f33514854f5bfa86d&output=dev&system=x86_64-darwin", + "https://cache.nixos.org/nar/115jv555r5lnkgb2hp9p8qz31h5s1a4sgjdkbaf0cp9ccb6lvl4y.nar.zst", + ), + ( + "pkg:nix/nixpkgs/SDL_mixer@1.2.12?commit=3d46470bb3030020f7e1361f33514854f5bfa86d&output=out&system=x86_64-linux", + "https://cache.nixos.org/nar/0phnvv0k4bnfb4mnjhldh4rksqy2pxsg53n6nh8zd7ikixq4q2qi.nar.zst", + ), + ( + "pkg:nix/nixpkgs/SDL_mixer@1.2.12?commit=3d46470bb3030020f7e1361f33514854f5bfa86d&output=dev&system=x86_64-linux", + "https://cache.nixos.org/nar/1m1jh8jg98i0czg5hdiaa9yjmz48f0ldx8vwr2hbmlz65pcmrlls.nar.zst", + ), + ] + + self.assertEqual(len(packages), len(expected_purl_data)) + + for package, (expected_purl, expected_url) in zip(packages, expected_purl_data): + self.assertEqual(expected_purl, package.purl) + self.assertEqual(expected_url, package.download_url) diff --git a/minecode/tests/testfiles/nix/SDL_mixer_package-expected.json b/minecode/tests/testfiles/nix/SDL_mixer_package-expected.json new file mode 100644 index 00000000..c0a513a4 --- /dev/null +++ b/minecode/tests/testfiles/nix/SDL_mixer_package-expected.json @@ -0,0 +1,92 @@ +{ + "name": "SDL_mixer", + "summary": "SDL multi-channel audio mixer library", + "homepage_url": "http://www.libsdl.org/projects/SDL_mixer/", + "license": "Zlib", + "releases": [ + { + "version": "1.2.12", + "last_updated": "2026-06-27T07:37:20Z", + "platforms": [ + { + "arch": "arm64", + "os": "Linux", + "system": "aarch64-linux", + "attribute_path": "SDL_mixer", + "commit_hash": "3d46470bb3030020f7e1361f33514854f5bfa86d", + "date": "2026-06-27T07:37:20Z", + "outputs": [ + { + "name": "out", + "path": "/nix/store/bgg1d95f9px23i9db6aal7cjbkdw025n-SDL_mixer-1.2.12", + "default": true + }, + { + "name": "dev", + "path": "/nix/store/756a2pb6ynq0dr54wxbc25lsbm685v80-SDL_mixer-1.2.12-dev" + } + ] + }, + { + "arch": "arm64", + "os": "macOS", + "system": "aarch64-darwin", + "attribute_path": "SDL_mixer", + "commit_hash": "3d46470bb3030020f7e1361f33514854f5bfa86d", + "date": "2026-06-27T07:37:20Z", + "outputs": [ + { + "name": "out", + "path": "/nix/store/08l3d3zr073c2qxjy36642nd06jvykc0-SDL_mixer-1.2.12", + "default": true + }, + { + "name": "dev", + "path": "/nix/store/5pndmp41940jv4w6y0y9m6hpa3c7b61y-SDL_mixer-1.2.12-dev" + } + ] + }, + { + "arch": "x86-64", + "os": "macOS", + "system": "x86_64-darwin", + "attribute_path": "SDL_mixer", + "commit_hash": "3d46470bb3030020f7e1361f33514854f5bfa86d", + "date": "2026-06-27T07:37:20Z", + "outputs": [ + { + "name": "out", + "path": "/nix/store/wb44wp28lx6jjdxdj1x384v02jkff5nm-SDL_mixer-1.2.12", + "default": true + }, + { + "name": "dev", + "path": "/nix/store/kfz0d0r4mk70h8lvpyhr1zcxyiyld28i-SDL_mixer-1.2.12-dev" + } + ] + }, + { + "arch": "x86-64", + "os": "Linux", + "system": "x86_64-linux", + "attribute_path": "SDL_mixer", + "commit_hash": "3d46470bb3030020f7e1361f33514854f5bfa86d", + "date": "2026-06-27T07:37:20Z", + "outputs": [ + { + "name": "out", + "path": "/nix/store/4vpgs9snpddn62nqy1lyd3cr5wfa89di-SDL_mixer-1.2.12", + "default": true + }, + { + "name": "dev", + "path": "/nix/store/kgzknkqwjywvf243dsrkl08d85ff06f8-SDL_mixer-1.2.12-dev" + } + ] + } + ], + "platforms_summary": "Linux and macOS", + "outputs_summary": "out, dev" + } + ] +}