diff --git a/Makefile b/Makefile index 294b5726..0e35f5a7 100644 --- a/Makefile +++ b/Makefile @@ -10,8 +10,7 @@ # Python version can be specified with `$ PYTHON_EXE=python3.x make conf` PYTHON_EXE?=python3 VENV=venv -MANAGE=${VENV}/bin/python manage_purldb.py -MATCHCODE_MANAGE=${VENV}/bin/python manage_matchcode.py +MANAGE=${VENV}/bin/python manage.py ACTIVATE?=. ${VENV}/bin/activate; VIRTUALENV_PYZ=../etc/thirdparty/virtualenv.pyz # Do not depend on Python to generate the SECRET_KEY @@ -22,12 +21,9 @@ ENV_FILE=.env # Customize with `$ make postgres PACKAGEDB_DB_PASSWORD=YOUR_PASSWORD` PACKAGEDB_DB_PASSWORD=packagedb -MATCHCODEIO_DB_PASSWORD=matchcodeio -SCANCODEIO_DB_PASSWORD=scancodeio # Django settings shortcuts -DJSM_PDB=DJANGO_SETTINGS_MODULE=purldb_project.settings -DJSM_MAT=DJANGO_SETTINGS_MODULE=matchcode_project.settings +DJSM_PDB=DJANGO_SETTINGS_MODULE=purldb.settings # Use sudo for postgres, but only on Linux UNAME := $(shell uname) @@ -58,8 +54,6 @@ envfile: envfile_testing: envfile @echo PACKAGEDB_DB_USER=\"postgres\" >> ${ENV_FILE} @echo PACKAGEDB_DB_PASSWORD=\"postgres\" >> ${ENV_FILE} - @echo SCANCODEIO_DB_USER=\"postgres\" >> ${ENV_FILE} - @echo SCANCODEIO_DB_PASSWORD=\"postgres\" >> ${ENV_FILE} doc8: @echo "-> Run doc8 validation" @@ -101,23 +95,9 @@ postgres: ${SUDO_POSTGRES} createdb --encoding=utf-8 --owner=packagedb packagedb @$(MAKE) migrate -postgres_matchcodeio: - @echo "-> Configure PostgreSQL database" - @echo "-> Create database user 'matchcodeio'" - ${SUDO_POSTGRES} createuser --no-createrole --no-superuser --login --inherit --createdb matchcodeio || true - ${SUDO_POSTGRES} psql -c "alter user matchcodeio with encrypted password '${MATCHCODEIO_DB_PASSWORD}';" || true - @echo "-> Drop 'matchcodeio' database" - ${SUDO_POSTGRES} dropdb matchcodeio || true - @echo "-> Create 'matchcodeio' database" - ${SUDO_POSTGRES} createdb --encoding=utf-8 --owner=matchcodeio matchcodeio - ${MATCHCODE_MANAGE} migrate - run: ${MANAGE} runserver 8001 --insecure -run_matchcodeio: - ${MATCHCODE_MANAGE} runserver 8002 --insecure - seed: ${MANAGE} seed @@ -128,20 +108,14 @@ run_map: ${MANAGE} run_map test_purldb: - ${ACTIVATE} ${DJSM_PDB} pytest -vvs --lf minecode packagedb purl2vcs purldb_project purldb_public_project --ignore packagedb/tests/test_throttling.py + ${ACTIVATE} ${DJSM_PDB} pytest -vvs --lf minecode matchcode packagedb purl2vcs purldb clearcode clearindex --ignore packagedb/tests/test_throttling.py ${ACTIVATE} ${DJSM_PDB} pytest -vvs --lf packagedb/tests/test_throttling.py -test_clearcode: - ${ACTIVATE} ${DJSM_PDB} ${PYTHON_EXE} -m pytest -vvs clearcode clearindex - -test_matchcode: - ${ACTIVATE} ${DJSM_MAT} ${PYTHON_EXE} -m pytest -vvs matchcode_pipeline matchcode - -test: test_purldb test_matchcode test_toolkit test_clearcode - -test_minecode: +test_minecode_pipelines: ${ACTIVATE} ${PYTHON_EXE} -m pytest -vvs minecode_pipelines +test: test_purldb test_minecode_pipelines + shell: ${MANAGE} shell @@ -179,4 +153,4 @@ docker-images: @docker save minecode minecode_minecode nginx | gzip > dist/minecode-images-`git describe --tags`.tar.gz # keep this sorted -.PHONY: bump check docs-check clean clearindex clearsync conf dev docker-images docs envfile envfile_testing index_packages migrate postgres postgres_matchcodeio priority_queue run run_map run_matchcodeio run_visit seed shell test test_clearcode test_matchcode test_purldb test_toolkit valid virtualenv +.PHONY: bump check docs-check clean clearindex clearsync conf dev docker-images docs envfile envfile_testing index_packages migrate postgres priority_queue run run_map run_visit seed shell test test_purldb test_minecode_pipelines valid virtualenv diff --git a/docker-compose.common.yml b/docker-compose.common.yml index a0256022..e31f4af1 100644 --- a/docker-compose.common.yml +++ b/docker-compose.common.yml @@ -18,9 +18,9 @@ services: web: build: . command: wait-for-it --strict --timeout=60 db:5432 -- sh -c " - python manage_purldb.py migrate && - python manage_purldb.py collectstatic --no-input --verbosity 0 --clear && - gunicorn purldb_project.wsgi:application --bind :8000 --timeout 600 --workers 8" + python manage.py migrate && + python manage.py collectstatic --no-input --verbosity 0 --clear && + gunicorn purldb.wsgi:application --bind :8000 --timeout 600 --workers 8" env_file: - docker_purldb.env expose: @@ -35,7 +35,7 @@ services: priority_queue: build: . - command: wait-for-it web:8000 -- python manage_purldb.py priority_queue + command: wait-for-it web:8000 -- python manage.py priority_queue env_file: - docker_purldb.env volumes: @@ -49,7 +49,7 @@ services: scheduler: build: . - command: wait-for-it web:8000 -- python manage_purldb.py run_scheduler + command: wait-for-it web:8000 -- python manage.py run_scheduler env_file: - docker_purldb.env volumes: @@ -62,7 +62,7 @@ services: rq_worker: build: . - command: wait-for-it web:8000 -- python manage_purldb.py rqworker default + command: wait-for-it web:8000 -- python manage.py rqworker default env_file: - docker_purldb.env volumes: diff --git a/docker-compose.matchcodeio.yml b/docker-compose.matchcodeio.yml deleted file mode 100644 index 210ec3b4..00000000 --- a/docker-compose.matchcodeio.yml +++ /dev/null @@ -1,86 +0,0 @@ -services: - matchcodeio_db: - image: postgres:13 - env_file: - - docker_matchcodeio.env - volumes: - - matchcodeio_db_data:/var/lib/postgresql/data/ - - matchcodeio_redis: - image: redis - # Enable redis data persistence using the "Append Only File" with the - # default policy of fsync every second. See https://redis.io/topics/persistence - command: redis-server --appendonly yes - volumes: - - redis_data:/data - restart: always - - matchcodeio_web: - build: . - command: wait-for-it --strict --timeout=60 db:5432 -- sh -c " - python manage_matchcode.py migrate --database default && - python manage_matchcode.py collectstatic --no-input --verbosity 0 --clear && - gunicorn matchcode_project.wsgi:application --bind :8001 --timeout 600 --workers 8" - environment: - - DJANGO_SETTINGS_MODULE=matchcode_project.settings - env_file: - - docker_matchcodeio.env - expose: - - 8001 - volumes: - - .env:/opt/scancodeio/.env - - /etc/scancodeio/:/etc/scancodeio/ - - workspace:/var/scancodeio/workspace/ - - static:/var/scancodeio/static/ - - matchcodeio_worker: - build: . - # Ensure that potential db migrations run first by waiting until "web" is up - command: wait-for-it --strict --timeout=120 matchcodeio_web:8001 -- sh -c " - python manage_matchcode.py rqworker --worker-class scancodeio.worker.ScanCodeIOWorker - --queue-class scancodeio.worker.ScanCodeIOQueue - --verbosity 1" - environment: - - DJANGO_SETTINGS_MODULE=matchcode_project.settings - env_file: - - docker_matchcodeio.env - volumes: - - .env:/opt/scancodeio/.env - - /etc/scancodeio/:/etc/scancodeio/ - - workspace:/var/scancodeio/workspace/ - depends_on: - - matchcodeio_redis - - matchcodeio_web - - matchcodeio_nginx: - image: nginx:alpine - volumes: - - ./etc/nginx/matchcodeio-conf.d/:/etc/nginx/conf.d/ - - /var/www/html:/var/www/html - - static:/var/scancodeio/static/ - depends_on: - - matchcodeio_web - restart: always - labels: - - "traefik.enable=true" - - "traefik.http.routers.matchcodeio.rule= - Host(`127.0.0.1`) && PathPrefix(`/api/matching`) - || Host(`127.0.0.1`) && PathPrefix(`/api/runs`) - || Host(`127.0.0.1`) && PathPrefix(`/project`) - || Host(`127.0.0.1`) && PathPrefix(`/runs`) - || Host(`localhost`) && PathPrefix(`/api/matching`) - || Host(`localhost`) && PathPrefix(`/api/runs`) - || Host(`localhost`) && PathPrefix(`/project`) - || Host(`localhost`) && PathPrefix(`/runs`)" - - "traefik.http.routers.matchcodeio.entrypoints=web" - -networks: - default: - name: purldb - external: true - -volumes: - redis_data: - static: - workspace: - matchcodeio_db_data: diff --git a/docker-compose.miners.yml b/docker-compose.miners.yml index ba33ccde..22b8b51f 100644 --- a/docker-compose.miners.yml +++ b/docker-compose.miners.yml @@ -5,8 +5,8 @@ services: visitor: build: . command: sh -c " - wait-for-it web:8000 -- python manage_purldb.py seed && - python manage_purldb.py run_visit --ignore-robots --ignore-throttle" + wait-for-it web:8000 -- python manage.py seed && + python manage.py run_visit --ignore-robots --ignore-throttle" env_file: - docker_purldb.env volumes: @@ -19,7 +19,7 @@ services: mapper: build: . - command: wait-for-it web:8000 -- python manage_purldb.py run_map + command: wait-for-it web:8000 -- python manage.py run_map env_file: - docker_purldb.env volumes: @@ -45,7 +45,7 @@ services: clearindex: build: . - command: wait-for-it web:8000 -- python manage_purldb.py run_clearindex + command: wait-for-it web:8000 -- python manage.py run_clearindex env_file: - docker_purldb.env profiles: diff --git a/docs/source/conf.py b/docs/source/conf.py index 1f94d7cf..6e146106 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -12,7 +12,7 @@ import os import sys -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "purldb_project.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "purldb.settings") sys.path.insert(0, os.path.abspath("../../.")) # -- Project information ----------------------------------------------------- diff --git a/docs/source/how-to-guides/purl_watch_how_to.rst b/docs/source/how-to-guides/purl_watch_how_to.rst index 40e8d0a7..2c089830 100644 --- a/docs/source/how-to-guides/purl_watch_how_to.rst +++ b/docs/source/how-to-guides/purl_watch_how_to.rst @@ -15,7 +15,7 @@ following management command on a periodic basis: .. code-block:: bash - python manage_purldb.py watch_packages + python manage.py watch_packages This command checks all PURLs in your PurlDB for new versions and indexes them accordingly. diff --git a/docs/source/purldb/purl_watch.rst b/docs/source/purldb/purl_watch.rst index 12f95697..3325a32a 100644 --- a/docs/source/purldb/purl_watch.rst +++ b/docs/source/purldb/purl_watch.rst @@ -16,7 +16,7 @@ Using the Management Command ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ For a relatively small and focused PurlDB, one can use the management -command ``python manage_purldb.py watch_packages``. This command can be +command ``python manage.py watch_packages``. This command can be run periodically using a cron job to watch all the PURLs in your PurlDB for new versions. Upon detecting new versions, it collects and indexes the new package. This approach is efficient for smaller databases. diff --git a/manage_purldb.py b/manage.py similarity index 87% rename from manage_purldb.py rename to manage.py index 62029b84..d3de09bc 100755 --- a/manage_purldb.py +++ b/manage.py @@ -14,5 +14,5 @@ if __name__ == "__main__": from django.core.management import execute_from_command_line - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "purldb_project.settings") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "purldb.settings") execute_from_command_line(sys.argv) diff --git a/manage_matchcode.py b/manage_matchcode.py deleted file mode 100755 index bfaed621..00000000 --- a/manage_matchcode.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import os -import sys - -if __name__ == "__main__": - from django.core.management import execute_from_command_line - - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "matchcode_project.settings") - execute_from_command_line(sys.argv) diff --git a/matchcode/api.py b/matchcode/api.py index bc2dc7fa..63f5228b 100644 --- a/matchcode/api.py +++ b/matchcode/api.py @@ -6,15 +6,25 @@ # See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # +from uuid import uuid4 + from django.db.models import Q from django.forms import widgets from django.forms.fields import MultipleChoiceField from django_filters.filters import MultipleChoiceFilter from django_filters.rest_framework import FilterSet +from matchcode.models import ApproximateDirectoryContentIndex +from matchcode.models import ApproximateDirectoryStructureIndex +from matchcode.models import ExactFileIndex +from matchcode.models import ExactPackageArchiveIndex from matchcode_toolkit.fingerprinting import create_halohash_chunks from matchcode_toolkit.fingerprinting import hexstring_to_binarray from matchcode_toolkit.fingerprinting import split_fingerprint +from rest_framework import mixins +from rest_framework import renderers +from rest_framework import serializers +from rest_framework import viewsets from rest_framework.decorators import action from rest_framework.response import Response from rest_framework.serializers import CharField @@ -25,12 +35,18 @@ from rest_framework.serializers import Serializer from rest_framework.viewsets import ReadOnlyModelViewSet from samecode.halohash import byte_hamming_distance - -from matchcode.models import ApproximateDirectoryContentIndex -from matchcode.models import ApproximateDirectoryStructureIndex -from matchcode.models import ExactFileIndex -from matchcode.models import ExactPackageArchiveIndex - +from scanpipe.api import ExcludeFromListViewMixin +from scanpipe.api.serializers import InputSourceSerializer +from scanpipe.api.serializers import SerializerExcludeFieldsMixin +from scanpipe.api.serializers import StrListField +from scanpipe.api.views import ProjectFilterSet +from scanpipe.api.views import RunViewSet +from scanpipe.models import Project +from scanpipe.models import Run +from scanpipe.pipes import count_group_by +from scanpipe.pipes.fetch import check_urls_availability +from scanpipe.pipes.fetch import fetch_urls +from scanpipe.views import project_results_json_response class BaseFileIndexSerializer(ModelSerializer): sha1 = CharField(source="fingerprint") @@ -285,3 +301,379 @@ class ApproximateDirectoryStructureIndexViewSet(BaseDirectoryIndexViewSet): queryset = ApproximateDirectoryStructureIndex.objects.all() serializer_class = ApproximateDirectoryStructureIndexSerializer filterset_class = ApproximateDirectoryStructureFilterSet + + +class RunSerializer(SerializerExcludeFieldsMixin, serializers.ModelSerializer): + project = serializers.HyperlinkedRelatedField(view_name="run-detail", read_only=True) + + class Meta: + model = Run + fields = [ + "url", + "pipeline_name", + "status", + "description", + "project", + "uuid", + "created_date", + "scancodeio_version", + "task_id", + "task_start_date", + "task_end_date", + "task_exitcode", + "task_output", + "log", + "execution_time", + ] + + +class RunViewSet(RunViewSet): + serializer_class = RunSerializer + + +class MatchingSerializer(ExcludeFromListViewMixin, serializers.ModelSerializer): + upload_file = serializers.FileField(write_only=True, required=False) + input_urls = StrListField( + write_only=True, + required=False, + style={"base_template": "textarea.html"}, + ) + webhook_url = serializers.CharField(write_only=True, required=False) + runs = RunSerializer(many=True, read_only=True) + input_sources = InputSourceSerializer( + source="inputsources", + many=True, + read_only=True, + ) + codebase_resources_summary = serializers.SerializerMethodField() + discovered_packages_summary = serializers.SerializerMethodField() + discovered_dependencies_summary = serializers.SerializerMethodField() + codebase_relations_summary = serializers.SerializerMethodField() + + class Meta: + model = Project + fields = ( + "url", + "uuid", + "upload_file", + "input_urls", + "webhook_url", + "created_date", + "input_sources", + "runs", + "resource_count", + "package_count", + "dependency_count", + "relation_count", + "codebase_resources_summary", + "discovered_packages_summary", + "discovered_dependencies_summary", + "codebase_relations_summary", + ) + exclude_from_list_view = [ + "resource_count", + "package_count", + "dependency_count", + "relation_count", + "codebase_resources_summary", + "discovered_packages_summary", + "discovered_dependencies_summary", + "codebase_relations_summary", + ] + extra_kwargs = { + "url": { + "view_name": "matching-detail", + "lookup_field": "pk", + }, + } + + def get_codebase_resources_summary(self, project): + queryset = project.codebaseresources.all() + return count_group_by(queryset, "status") + + def get_discovered_packages_summary(self, project): + base_qs = project.discoveredpackages + return { + "total": base_qs.count(), + "with_missing_resources": base_qs.exclude(missing_resources=[]).count(), + "with_modified_resources": base_qs.exclude(modified_resources=[]).count(), + } + + def get_discovered_dependencies_summary(self, project): + base_qs = project.discovereddependencies + return { + "total": base_qs.count(), + "is_runtime": base_qs.filter(is_runtime=True).count(), + "is_optional": base_qs.filter(is_optional=True).count(), + "is_pinned": base_qs.filter(is_pinned=True).count(), + } + + def get_codebase_relations_summary(self, project): + queryset = project.codebaserelations.all() + return count_group_by(queryset, "map_type") + + def validate_input_urls(self, value): + """Add support for providing multiple URLs in a single string.""" + return [url for entry in value for url in entry.split()] + + def create(self, validated_data, matching_pipeline_name="matching"): + """Create a new `project` with `upload_file`, using the `matching` pipeline""" + execute_now = True + validated_data["name"] = uuid4() + upload_file = validated_data.pop("upload_file", None) + input_urls = validated_data.pop("input_urls", []) + webhook_url = validated_data.pop("webhook_url", None) + + downloads, errors = fetch_urls(input_urls) + if errors: + raise serializers.ValidationError("Could not fetch: " + "\n".join(errors)) + + project = super().create(validated_data) + + if upload_file: + project.add_uploads([upload_file]) + + if downloads: + project.add_downloads(downloads) + + if webhook_url: + project.add_webhook_subscription(webhook_url) + + project.add_pipeline(matching_pipeline_name, execute_now) + + return project + + +class D2DSerializer(ExcludeFromListViewMixin, serializers.ModelSerializer): + input_urls = StrListField( + write_only=True, + required=True, + style={"base_template": "textarea.html"}, + ) + + codebase_resources_summary = serializers.SerializerMethodField() + discovered_packages_summary = serializers.SerializerMethodField() + discovered_dependencies_summary = serializers.SerializerMethodField() + codebase_relations_summary = serializers.SerializerMethodField() + codebase_resources_discrepancies = serializers.SerializerMethodField() + + class Meta: + model = Project + fields = ( + "url", + "uuid", + "input_urls", + "created_date", + "input_sources", + "runs", + "resource_count", + "package_count", + "dependency_count", + "relation_count", + "codebase_resources_summary", + "discovered_packages_summary", + "discovered_dependencies_summary", + "codebase_relations_summary", + "codebase_resources_discrepancies", + ) + exclude_from_list_view = [ + "resource_count", + "package_count", + "dependency_count", + "relation_count", + "codebase_resources_summary", + "discovered_packages_summary", + "discovered_dependencies_summary", + "codebase_relations_summary", + "codebase_resources_discrepancies", + ] + extra_kwargs = { + "url": { + "view_name": "d2d-detail", + "lookup_field": "pk", + }, + } + + def get_codebase_resources_summary(self, project): + queryset = project.codebaseresources.all() + return count_group_by(queryset, "status") + + def get_codebase_resources_discrepancies(self, project): + queryset = project.codebaseresources.filter(status="requires-review") + return { + "total": queryset.count(), + } + + def get_discovered_packages_summary(self, project): + base_qs = project.discoveredpackages + return { + "total": base_qs.count(), + "with_missing_resources": base_qs.exclude(missing_resources=[]).count(), + "with_modified_resources": base_qs.exclude(modified_resources=[]).count(), + } + + def get_discovered_dependencies_summary(self, project): + base_qs = project.discovereddependencies + return { + "total": base_qs.count(), + "is_runtime": base_qs.filter(is_runtime=True).count(), + "is_optional": base_qs.filter(is_optional=True).count(), + "is_pinned": base_qs.filter(is_pinned=True).count(), + } + + def get_codebase_relations_summary(self, project): + queryset = project.codebaserelations.all() + return count_group_by(queryset, "map_type") + + def create(self, validated_data, matching_pipeline_name="d2d"): + """Create a new `project` with `input_urls`, using the `d2d` pipeline""" + execute_now = True + validated_data["name"] = uuid4() + input_urls = validated_data.pop("input_urls", []) + errors = check_urls_availability(input_urls) + + if errors: + raise serializers.ValidationError("Could not fetch: " + "\n".join(errors)) + + project = super().create(validated_data) + + urls = [] + + for url in input_urls: + value = url + if "\n" in value: + input_urls = input_urls[0].split("\n") + input_urls = [x.strip() for x in input_urls] + input_urls = list(filter(None, input_urls)) + urls.extend(input_urls) + else: + value = value.strip() + if value: + urls.append(value) + + for url in urls: + project.add_input_source(download_url=url) + + project.add_pipeline( + matching_pipeline_name, + selected_groups=["Java", "Javascript", "Elf", "Go"], + execute_now=execute_now, + ) + + return project + + +class MatchingViewSet( + mixins.CreateModelMixin, + mixins.RetrieveModelMixin, + mixins.DestroyModelMixin, + mixins.ListModelMixin, + viewsets.GenericViewSet, +): + """ + Take a ScanCode.io JSON of a codebase `upload_file` or from a list of + `input_urls` and run the ``matching`` pipeline + (https://github.com/aboutcode-org/purldb/blob/main/matchcode_pipeline/pipelines/matching.py) + on it. + + The ``matching`` pipeline matches directory and resources of the codebase in + ``upload_file`` to Packages indexed in the PurlDB. + + **Request example:** + + { + "input_urls": + } + + Then return a mapping containing information about the match request: + + - url + - URL of the match request + - uuid + - UUID of the match request + - created_date + - Date of the match request + - input_sources + - List of input files for the match request + - runs + - List of mapping containing details about the runs created for this + match request. + """ + + queryset = Project.objects.all() + serializer_class = MatchingSerializer + filterset_class = ProjectFilterSet + + def get_queryset(self): + return ( + super() + .get_queryset() + .prefetch_related( + "runs", + ) + ) + + @action(detail=True, renderer_classes=[renderers.JSONRenderer]) + def results(self, request, *args, **kwargs): + """ + Return the results compatible with ScanCode data format. + The content is returned as a stream of JSON content using the + JSONResultsGenerator class. + """ + return project_results_json_response(self.get_object()) + + +class D2DViewSet( + mixins.CreateModelMixin, + mixins.RetrieveModelMixin, + mixins.DestroyModelMixin, + mixins.ListModelMixin, + viewsets.GenericViewSet, +): + """ + Take a list of `input_urls` containing package download urls and match it to its source. + + **Request example:** + + { + "input_urls": [ + "https://registry.npmjs.com/asdf/-/asdf-1.0.2.tgz" + ] + } + + Then return a mapping containing information about the match request: + + - url + - URL of the match request + - uuid + - UUID of the match request + - created_date + - Date of the match request + - input_sources + - List of input files for the match request + - runs + - List of mapping containing details about the runs created for this + match request. + """ + + queryset = Project.objects.all() + serializer_class = D2DSerializer + filterset_class = ProjectFilterSet + + def get_queryset(self): + return ( + super() + .get_queryset() + .prefetch_related( + "runs", + ) + ) + + @action(detail=True, renderer_classes=[renderers.JSONRenderer]) + def results(self, request, *args, **kwargs): + """ + Return the results compatible with ScanCode data format. + The content is returned as a stream of JSON content using the + JSONResultsGenerator class. + """ + return project_results_json_response(self.get_object()) diff --git a/matchcode_pipeline/__init__.py b/matchcode/pipelines/__init__.py similarity index 100% rename from matchcode_pipeline/__init__.py rename to matchcode/pipelines/__init__.py diff --git a/matchcode_pipeline/pipelines/matching.py b/matchcode/pipelines/matching.py similarity index 99% rename from matchcode_pipeline/pipelines/matching.py rename to matchcode/pipelines/matching.py index 6cdd857b..b8e7e5d5 100644 --- a/matchcode_pipeline/pipelines/matching.py +++ b/matchcode/pipelines/matching.py @@ -24,7 +24,7 @@ from scanpipe.pipelines.scan_codebase import ScanCodebase from scanpipe.pipes import matchcode -from matchcode_pipeline.pipes import matching +from matchcode.pipes import matching class Matching(ScanCodebase, LoadInventory): diff --git a/matchcode_pipeline/pipelines/__init__.py b/matchcode/pipes/__init__.py similarity index 100% rename from matchcode_pipeline/pipelines/__init__.py rename to matchcode/pipes/__init__.py diff --git a/matchcode_pipeline/pipes/matching.py b/matchcode/pipes/matching.py similarity index 99% rename from matchcode_pipeline/pipes/matching.py rename to matchcode/pipes/matching.py index 4f9f3fa7..3a1d6e16 100644 --- a/matchcode_pipeline/pipes/matching.py +++ b/matchcode/pipes/matching.py @@ -112,7 +112,7 @@ def match_purldb_package(project, resources_by_sha1, enhance_package_data=True, """ match_count = 0 sha1_list = list(resources_by_sha1.keys()) - results = Package.objects.using("packagedb").filter(sha1__in=sha1_list).order_by() + results = Package.objects.filter(sha1__in=sha1_list).order_by() # Process matched Package data for package in results: package_data = package.to_dict() @@ -145,7 +145,7 @@ def match_purldb_resource(project, resources_by_sha1, package_data_by_purldb_url match_count = 0 sha1_list = list(resources_by_sha1.keys()) results = ( - Resource.objects.using("packagedb") + Resource.objects .filter(sha1__in=sha1_list) .select_related("package") .only("package__uuid") diff --git a/matchcode_pipeline/tests/test_api.py b/matchcode/tests/test_api.py similarity index 93% rename from matchcode_pipeline/tests/test_api.py rename to matchcode/tests/test_api.py index 98adc981..10524de0 100644 --- a/matchcode_pipeline/tests/test_api.py +++ b/matchcode/tests/test_api.py @@ -26,8 +26,7 @@ class MatchCodePipelineAPITest(TransactionTestCase): - databases = {"default", "packagedb"} - data_location = Path(__file__).parent / "data" + data_location = Path(__file__).parent / "testfiles" / "pipeline" def setUp(self): self.project1 = Project.objects.create(name="Analysis") @@ -46,8 +45,8 @@ def setUp(self): map_type="java_to_class", ) - self.matching_list_url = reverse("matching-list") - self.project1_detail_url = reverse("matching-detail", args=[self.project1.uuid]) + self.matching_list_url = reverse("api:matching-list") + self.project1_detail_url = reverse("api:matching-detail", args=[self.project1.uuid]) self.user = User.objects.create_user("username", "e@mail.com", "secret") self.auth = f"Token {self.user.auth_token.key}" @@ -172,8 +171,8 @@ def test_matching_pipeline_api_matching_create_multiple_input_urls( def test_matchcode_pipeline_api_run_detail(self): run1 = self.project1.add_pipeline("matching") - url = reverse("run-detail", args=[run1.uuid]) - project1_detail_url = reverse("run-detail", args=[self.project1.uuid]) + url = reverse("api:run-detail", args=[run1.uuid]) + project1_detail_url = reverse("api:run-detail", args=[self.project1.uuid]) response = self.csrf_client.get(url) self.assertEqual(str(run1.uuid), response.data["uuid"]) self.assertIn(project1_detail_url, response.data["project"]) @@ -189,13 +188,12 @@ def test_matchcode_pipeline_api_run_detail(self): class D2DPipelineAPITest(TransactionTestCase): - databases = {"default", "packagedb"} data_location = Path(__file__).parent / "data" def setUp(self): self.project1 = Project.objects.create(name="Analysis") - self.d2d_list_url = reverse("d2d-list") - self.project1_detail_url = reverse("d2d-detail", args=[self.project1.uuid]) + self.d2d_list_url = reverse("api:d2d-list") + self.project1_detail_url = reverse("api:d2d-detail", args=[self.project1.uuid]) self.user = User.objects.create_user("username", "a@mail.com", "secret") self.auth = f"Token {self.user.auth_token.key}" @@ -234,8 +232,8 @@ def test_d2d_pipeline_api_d2d_create(self, mock_execute_pipeline_task): def test_d2d_pipeline_api_run_detail(self): run1 = self.project1.add_pipeline("d2d") - url = reverse("run-detail", args=[run1.uuid]) - project1_detail_url = reverse("run-detail", args=[self.project1.uuid]) + url = reverse("api:run-detail", args=[run1.uuid]) + project1_detail_url = reverse("api:run-detail", args=[self.project1.uuid]) response = self.csrf_client.get(url) self.assertEqual(str(run1.uuid), response.data["uuid"]) self.assertIn(project1_detail_url, response.data["project"]) diff --git a/matchcode_pipeline/tests/data/d2d-javascript/to/main.js b/matchcode/tests/testfiles/pipeline/d2d-javascript/to/main.js similarity index 100% rename from matchcode_pipeline/tests/data/d2d-javascript/to/main.js rename to matchcode/tests/testfiles/pipeline/d2d-javascript/to/main.js diff --git a/matchcode_pipeline/tests/data/d2d-javascript/to/main.js.map b/matchcode/tests/testfiles/pipeline/d2d-javascript/to/main.js.map similarity index 100% rename from matchcode_pipeline/tests/data/d2d-javascript/to/main.js.map rename to matchcode/tests/testfiles/pipeline/d2d-javascript/to/main.js.map diff --git a/matchcode_pipeline/tests/data/d2d/find_java_packages/Foo.java b/matchcode/tests/testfiles/pipeline/d2d/find_java_packages/Foo.java similarity index 100% rename from matchcode_pipeline/tests/data/d2d/find_java_packages/Foo.java rename to matchcode/tests/testfiles/pipeline/d2d/find_java_packages/Foo.java diff --git a/matchcode_pipeline/tests/data/match-creation-api-results.json b/matchcode/tests/testfiles/pipeline/match-creation-api-results.json similarity index 100% rename from matchcode_pipeline/tests/data/match-creation-api-results.json rename to matchcode/tests/testfiles/pipeline/match-creation-api-results.json diff --git a/matchcode_pipeline/tests/data/test-out.json b/matchcode/tests/testfiles/pipeline/test-out.json similarity index 100% rename from matchcode_pipeline/tests/data/test-out.json rename to matchcode/tests/testfiles/pipeline/test-out.json diff --git a/matchcode_pipeline/api.py b/matchcode_pipeline/api.py deleted file mode 100644 index fb3672fb..00000000 --- a/matchcode_pipeline/api.py +++ /dev/null @@ -1,403 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -from uuid import uuid4 - -from rest_framework import mixins -from rest_framework import renderers -from rest_framework import serializers -from rest_framework import viewsets -from rest_framework.decorators import action -from scanpipe.api import ExcludeFromListViewMixin -from scanpipe.api.serializers import InputSourceSerializer -from scanpipe.api.serializers import SerializerExcludeFieldsMixin -from scanpipe.api.serializers import StrListField -from scanpipe.api.views import ProjectFilterSet -from scanpipe.api.views import RunViewSet -from scanpipe.models import Project -from scanpipe.models import Run -from scanpipe.pipes import count_group_by -from scanpipe.pipes.fetch import check_urls_availability -from scanpipe.pipes.fetch import fetch_urls -from scanpipe.views import project_results_json_response - - -class RunSerializer(SerializerExcludeFieldsMixin, serializers.ModelSerializer): - project = serializers.HyperlinkedRelatedField(view_name="run-detail", read_only=True) - - class Meta: - model = Run - fields = [ - "url", - "pipeline_name", - "status", - "description", - "project", - "uuid", - "created_date", - "scancodeio_version", - "task_id", - "task_start_date", - "task_end_date", - "task_exitcode", - "task_output", - "log", - "execution_time", - ] - - -class RunViewSet(RunViewSet): - serializer_class = RunSerializer - - -class MatchingSerializer(ExcludeFromListViewMixin, serializers.ModelSerializer): - upload_file = serializers.FileField(write_only=True, required=False) - input_urls = StrListField( - write_only=True, - required=False, - style={"base_template": "textarea.html"}, - ) - webhook_url = serializers.CharField(write_only=True, required=False) - runs = RunSerializer(many=True, read_only=True) - input_sources = InputSourceSerializer( - source="inputsources", - many=True, - read_only=True, - ) - codebase_resources_summary = serializers.SerializerMethodField() - discovered_packages_summary = serializers.SerializerMethodField() - discovered_dependencies_summary = serializers.SerializerMethodField() - codebase_relations_summary = serializers.SerializerMethodField() - - class Meta: - model = Project - fields = ( - "url", - "uuid", - "upload_file", - "input_urls", - "webhook_url", - "created_date", - "input_sources", - "runs", - "resource_count", - "package_count", - "dependency_count", - "relation_count", - "codebase_resources_summary", - "discovered_packages_summary", - "discovered_dependencies_summary", - "codebase_relations_summary", - ) - exclude_from_list_view = [ - "resource_count", - "package_count", - "dependency_count", - "relation_count", - "codebase_resources_summary", - "discovered_packages_summary", - "discovered_dependencies_summary", - "codebase_relations_summary", - ] - extra_kwargs = { - "url": { - "view_name": "matching-detail", - "lookup_field": "pk", - }, - } - - def get_codebase_resources_summary(self, project): - queryset = project.codebaseresources.all() - return count_group_by(queryset, "status") - - def get_discovered_packages_summary(self, project): - base_qs = project.discoveredpackages - return { - "total": base_qs.count(), - "with_missing_resources": base_qs.exclude(missing_resources=[]).count(), - "with_modified_resources": base_qs.exclude(modified_resources=[]).count(), - } - - def get_discovered_dependencies_summary(self, project): - base_qs = project.discovereddependencies - return { - "total": base_qs.count(), - "is_runtime": base_qs.filter(is_runtime=True).count(), - "is_optional": base_qs.filter(is_optional=True).count(), - "is_pinned": base_qs.filter(is_pinned=True).count(), - } - - def get_codebase_relations_summary(self, project): - queryset = project.codebaserelations.all() - return count_group_by(queryset, "map_type") - - def validate_input_urls(self, value): - """Add support for providing multiple URLs in a single string.""" - return [url for entry in value for url in entry.split()] - - def create(self, validated_data, matching_pipeline_name="matching"): - """Create a new `project` with `upload_file`, using the `matching` pipeline""" - execute_now = True - validated_data["name"] = uuid4() - upload_file = validated_data.pop("upload_file", None) - input_urls = validated_data.pop("input_urls", []) - webhook_url = validated_data.pop("webhook_url", None) - - downloads, errors = fetch_urls(input_urls) - if errors: - raise serializers.ValidationError("Could not fetch: " + "\n".join(errors)) - - project = super().create(validated_data) - - if upload_file: - project.add_uploads([upload_file]) - - if downloads: - project.add_downloads(downloads) - - if webhook_url: - project.add_webhook_subscription(webhook_url) - - project.add_pipeline(matching_pipeline_name, execute_now) - - return project - - -class D2DSerializer(ExcludeFromListViewMixin, serializers.ModelSerializer): - input_urls = StrListField( - write_only=True, - required=True, - style={"base_template": "textarea.html"}, - ) - - codebase_resources_summary = serializers.SerializerMethodField() - discovered_packages_summary = serializers.SerializerMethodField() - discovered_dependencies_summary = serializers.SerializerMethodField() - codebase_relations_summary = serializers.SerializerMethodField() - codebase_resources_discrepancies = serializers.SerializerMethodField() - - class Meta: - model = Project - fields = ( - "url", - "uuid", - "input_urls", - "created_date", - "input_sources", - "runs", - "resource_count", - "package_count", - "dependency_count", - "relation_count", - "codebase_resources_summary", - "discovered_packages_summary", - "discovered_dependencies_summary", - "codebase_relations_summary", - "codebase_resources_discrepancies", - ) - exclude_from_list_view = [ - "resource_count", - "package_count", - "dependency_count", - "relation_count", - "codebase_resources_summary", - "discovered_packages_summary", - "discovered_dependencies_summary", - "codebase_relations_summary", - "codebase_resources_discrepancies", - ] - extra_kwargs = { - "url": { - "view_name": "d2d-detail", - "lookup_field": "pk", - }, - } - - def get_codebase_resources_summary(self, project): - queryset = project.codebaseresources.all() - return count_group_by(queryset, "status") - - def get_codebase_resources_discrepancies(self, project): - queryset = project.codebaseresources.filter(status="requires-review") - return { - "total": queryset.count(), - } - - def get_discovered_packages_summary(self, project): - base_qs = project.discoveredpackages - return { - "total": base_qs.count(), - "with_missing_resources": base_qs.exclude(missing_resources=[]).count(), - "with_modified_resources": base_qs.exclude(modified_resources=[]).count(), - } - - def get_discovered_dependencies_summary(self, project): - base_qs = project.discovereddependencies - return { - "total": base_qs.count(), - "is_runtime": base_qs.filter(is_runtime=True).count(), - "is_optional": base_qs.filter(is_optional=True).count(), - "is_pinned": base_qs.filter(is_pinned=True).count(), - } - - def get_codebase_relations_summary(self, project): - queryset = project.codebaserelations.all() - return count_group_by(queryset, "map_type") - - def create(self, validated_data, matching_pipeline_name="d2d"): - """Create a new `project` with `input_urls`, using the `d2d` pipeline""" - execute_now = True - validated_data["name"] = uuid4() - input_urls = validated_data.pop("input_urls", []) - errors = check_urls_availability(input_urls) - - if errors: - raise serializers.ValidationError("Could not fetch: " + "\n".join(errors)) - - project = super().create(validated_data) - - urls = [] - - for url in input_urls: - value = url - if "\n" in value: - input_urls = input_urls[0].split("\n") - input_urls = [x.strip() for x in input_urls] - input_urls = list(filter(None, input_urls)) - urls.extend(input_urls) - else: - value = value.strip() - if value: - urls.append(value) - - for url in urls: - project.add_input_source(download_url=url) - - project.add_pipeline( - matching_pipeline_name, - selected_groups=["Java", "Javascript", "Elf", "Go"], - execute_now=execute_now, - ) - - return project - - -class MatchingViewSet( - mixins.CreateModelMixin, - mixins.RetrieveModelMixin, - mixins.DestroyModelMixin, - mixins.ListModelMixin, - viewsets.GenericViewSet, -): - """ - Take a ScanCode.io JSON of a codebase `upload_file` or from a list of - `input_urls` and run the ``matching`` pipeline - (https://github.com/aboutcode-org/purldb/blob/main/matchcode_pipeline/pipelines/matching.py) - on it. - - The ``matching`` pipeline matches directory and resources of the codebase in - ``upload_file`` to Packages indexed in the PurlDB. - - **Request example:** - - { - "input_urls": - } - - Then return a mapping containing information about the match request: - - - url - - URL of the match request - - uuid - - UUID of the match request - - created_date - - Date of the match request - - input_sources - - List of input files for the match request - - runs - - List of mapping containing details about the runs created for this - match request. - """ - - queryset = Project.objects.all() - serializer_class = MatchingSerializer - filterset_class = ProjectFilterSet - - def get_queryset(self): - return ( - super() - .get_queryset() - .prefetch_related( - "runs", - ) - ) - - @action(detail=True, renderer_classes=[renderers.JSONRenderer]) - def results(self, request, *args, **kwargs): - """ - Return the results compatible with ScanCode data format. - The content is returned as a stream of JSON content using the - JSONResultsGenerator class. - """ - return project_results_json_response(self.get_object()) - - -class D2DViewSet( - mixins.CreateModelMixin, - mixins.RetrieveModelMixin, - mixins.DestroyModelMixin, - mixins.ListModelMixin, - viewsets.GenericViewSet, -): - """ - Take a list of `input_urls` containing package download urls and match it to its source. - - **Request example:** - - { - "input_urls": [ - "https://registry.npmjs.com/asdf/-/asdf-1.0.2.tgz" - ] - } - - Then return a mapping containing information about the match request: - - - url - - URL of the match request - - uuid - - UUID of the match request - - created_date - - Date of the match request - - input_sources - - List of input files for the match request - - runs - - List of mapping containing details about the runs created for this - match request. - """ - - queryset = Project.objects.all() - serializer_class = D2DSerializer - filterset_class = ProjectFilterSet - - def get_queryset(self): - return ( - super() - .get_queryset() - .prefetch_related( - "runs", - ) - ) - - @action(detail=True, renderer_classes=[renderers.JSONRenderer]) - def results(self, request, *args, **kwargs): - """ - Return the results compatible with ScanCode data format. - The content is returned as a stream of JSON content using the - JSONResultsGenerator class. - """ - return project_results_json_response(self.get_object()) diff --git a/matchcode_pipeline/pipes/__init__.py b/matchcode_pipeline/pipes/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/matchcode_pipeline/tests/__init__.py b/matchcode_pipeline/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/matchcode_pipeline/tests/pipes/__init__.py b/matchcode_pipeline/tests/pipes/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/matchcode_pipeline/tests/pipes/test_matching.py b/matchcode_pipeline/tests/pipes/test_matching.py deleted file mode 100644 index f09032fa..00000000 --- a/matchcode_pipeline/tests/pipes/test_matching.py +++ /dev/null @@ -1,222 +0,0 @@ -import io -import uuid -from pathlib import Path - -from django.test import TestCase - -from scanpipe import pipes -from scanpipe.models import Project -from scanpipe.pipes import flag -from scanpipe.pipes.input import copy_inputs -from scanpipe.tests import make_resource_directory -from scanpipe.tests import make_resource_file -from scanpipe.tests import package_data1 -from scanpipe.tests import package_data2 - -from matchcode.models import ApproximateDirectoryContentIndex -from matchcode.models import ApproximateResourceContentIndex -from matchcode_pipeline.pipes import matching -from packagedb.models import Package -from packagedb.models import Resource - - -class MatchingPipesTest(TestCase): - data_location = Path(__file__).parent.parent / "data" - databases = {"packagedb", "default"} - - def setUp(self): - self.project1 = Project.objects.create(name="Analysis") - self.package1 = Package.objects.create( - type=package_data1["type"], - namespace=package_data1["namespace"], - name=package_data1["name"], - version=package_data1["version"], - sha1="abcdef", - ) - self.directory_content_fingerprint1 = ApproximateDirectoryContentIndex.index( - fingerprint="00000003238f6ed2c218090d4da80b3b42160e69", - resource_path="package.jar-extract", - package=self.package1, - ) - self.directory1 = Resource.objects.create( - path="package.jar-extract", - package=self.package1, - ) - self.resource1 = Resource.objects.create( - path="inflate.c", size=55466, package=self.package1 - ) - self.resource_content_fingerprint1 = ApproximateResourceContentIndex.index( - fingerprint="000018fba23a49e4cd40718d1297be719e6564a4", - resource_path="inflate.c", - package=self.package1, - ) - - def test_matchcode_pipeline_pipes_matching_get_project_resources_qs(self): - package_resource = make_resource_file(self.project1, "package.jar", is_archive=True) - make_resource_directory(self.project1, "package.jar-extract/") - make_resource_file(self.project1, "package.jar-extract/foo.class") - - directory_resource = make_resource_directory(self.project1, "directory1") - make_resource_file(self.project1, "directory1/foo.txt") - - # This directory and its contents should not be returned - make_resource_directory(self.project1, "directory100") - make_resource_file(self.project1, "directory100/bar.txt") - - resources = [package_resource, directory_resource] - resources_qs = matching.get_project_resources_qs(self.project1, resources=resources) - expected_paths = [ - "package.jar", - "package.jar-extract/", - "package.jar-extract/foo.class", - "directory1", - "directory1/foo.txt", - ] - expected_qs = self.project1.codebaseresources.filter(path__in=expected_paths) - self.assertQuerySetEqual(expected_qs, resources_qs) - - def test_matchcode_pipeline_pipes_matching_match_purldb_resources(self): - to_1 = make_resource_file(self.project1, "package.jar", sha1="abcdef") - to_1.is_archive = True - to_1.save() - # The initial status will be updated to flag.MATCHED_TO_PURLDB_PACKAGE - to_2 = make_resource_file(self.project1, "package.jar-extract/a.class", status=flag.MAPPED) - to_3 = make_resource_file(self.project1, "package.jar-extract/b.class") - - buffer = io.StringIO() - matching.match_purldb_resources( - self.project1, - matcher_func=matching.match_purldb_package, - archives_only=True, - logger=buffer.write, - ) - expected = ( - "Matching 1 resources in PurlDB, using SHA13 resources matched in PurlDB using 1 SHA1s" - ) - self.assertEqual(expected, buffer.getvalue()) - - package = self.project1.discoveredpackages.get() - self.assertEqual(package_data1["name"], package.name) - - for resource in [to_1, to_2, to_3]: - resource.refresh_from_db() - self.assertEqual(flag.MATCHED_TO_PURLDB_PACKAGE, resource.status) - self.assertEqual(package, resource.discovered_packages.get()) - - def test_matchcode_pipeline_pipes_matching_match_purldb_directories(self): - to_1 = make_resource_directory( - self.project1, - "package.jar-extract", - extra_data={"directory_content": "00000003238f6ed2c218090d4da80b3b42160e69"}, - ) - to_2 = make_resource_file(self.project1, "package.jar-extract/a.class") - to_3 = make_resource_file(self.project1, "package.jar-extract/b.class") - - buffer = io.StringIO() - matching.match_purldb_directories( - self.project1, - logger=buffer.write, - ) - - expected = "Matching 1 directory against PurlDB1 directory matched in PurlDB" - self.assertEqual(expected, buffer.getvalue()) - - package = self.project1.discoveredpackages.get() - self.assertEqual(package_data1["name"], package.name) - - for resource in [to_1, to_2, to_3]: - resource.refresh_from_db() - self.assertEqual("matched-to-purldb-directory", resource.status) - self.assertEqual(package, resource.discovered_packages.get()) - - def test_matchcode_pipeline_pipes_matching_match_purldb_resources_post_process( - self, - ): - to_map = self.data_location / "d2d-javascript" / "to" / "main.js.map" - to_mini = self.data_location / "d2d-javascript" / "to" / "main.js" - to_dir = ( - self.project1.codebase_path / "project.tar.zst/modules/apps/adaptive-media/" - "adaptive-media-web-extract/src/main/resources/META-INF/resources/" - "adaptive_media/js" - ) - to_dir.mkdir(parents=True) - copy_inputs([to_map, to_mini], to_dir) - - pipes.collect_and_create_codebase_resources(self.project1) - - resources = self.project1.codebaseresources.filter( - path__startswith=( - "project.tar.zst/modules/apps/adaptive-media/" - "adaptive-media-web-extract/src/main/resources/META-INF/resources/" - "adaptive_media/js/main.js" - ) - ) - - mini_resource = self.project1.codebaseresources.filter( - path=( - "project.tar.zst/modules/apps/adaptive-media/" - "adaptive-media-web-extract/src/main/resources/META-INF/resources/" - "adaptive_media/js/main.js" - ) - ) - - dummy_package_data1 = package_data1.copy() - dummy_package_data1["uuid"] = uuid.uuid4() - package1, _ = matching.create_package_from_purldb_data( - self.project1, - resources, - dummy_package_data1, - flag.MATCHED_TO_PURLDB_RESOURCE, - ) - - dummy_package_data2 = package_data2.copy() - dummy_package_data2["uuid"] = uuid.uuid4() - package2, _ = matching.create_package_from_purldb_data( - self.project1, - mini_resource, - dummy_package_data2, - flag.MATCHED_TO_PURLDB_RESOURCE, - ) - - buffer = io.StringIO() - matching.match_purldb_resources_post_process( - self.project1, - logger=buffer.write, - ) - expected = f"Refining matching for 1 {flag.MATCHED_TO_PURLDB_RESOURCE} archives." - self.assertIn(expected, buffer.getvalue()) - - package1_resource_count = package1.codebase_resources.count() - package2_resource_count = package2.codebase_resources.count() - - self.assertEqual(2, package1_resource_count) - self.assertEqual(0, package2_resource_count) - - def test_matchcode_pipeline_pipes_matching_match_purldb_resource_approximately( - self, - ): - resource = make_resource_file( - self.project1, - "inflate.c", - size=55466, - extra_data={"halo1": "000018f4aa3a49e4cd40718d1297be519e6564a4"}, - ) - - buffer = io.StringIO() - matching.match_purldb_resources_approximately( - self.project1, - logger=buffer.write, - ) - - expected = ( - "Approximate matching 1 resource against PurlDB" - "1 resource approximately matched in PurlDB" - ) - self.assertEqual(expected, buffer.getvalue()) - - package = self.project1.discoveredpackages.get() - self.assertEqual(package_data1["name"], package.name) - - resource.refresh_from_db() - self.assertEqual("approximate-matched-to-purldb-resource", resource.status) - self.assertEqual(package, resource.discovered_packages.get()) diff --git a/matchcode_project/__init__.py b/matchcode_project/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/matchcode_project/dbrouter.py b/matchcode_project/dbrouter.py deleted file mode 100644 index 835a8f06..00000000 --- a/matchcode_project/dbrouter.py +++ /dev/null @@ -1,64 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -class PackageDBRouter: - app_labels = [ - "clearcode", - "clearindex", - "minecode", - "matchcode", - "packagedb", - ] - - def db_for_read(self, model, **hints): - if model._meta.app_label in self.app_labels: - return "packagedb" - return None - - def db_for_write(self, model, **hints): - if model._meta.app_label in self.app_labels: - return "packagedb" - return None - - def allow_relation(self, obj1, obj2, **hints): - if obj1._meta.app_label in self.app_labels or obj2._meta.app_label in self.app_labels: - return True - return None - - def allow_migrate(self, db, app_label, model_name=None, **hints): - if app_label in self.app_labels: - return db == "packagedb" - return None - - -class ScancodeIORouter: - app_labels = [ - "scanpipe", - ] - - def db_for_read(self, model, **hints): - if model._meta.app_label in self.app_labels: - return "default" - return None - - def db_for_write(self, model, **hints): - if model._meta.app_label in self.app_labels: - return "default" - return None - - def allow_relation(self, obj1, obj2, **hints): - if obj1._meta.app_label in self.app_labels or obj2._meta.app_label in self.app_labels: - return True - return None - - def allow_migrate(self, db, app_label, model_name=None, **hints): - if app_label in self.app_labels: - return db == "default" - return None diff --git a/matchcode_project/settings.py b/matchcode_project/settings.py deleted file mode 100644 index 47e9216c..00000000 --- a/matchcode_project/settings.py +++ /dev/null @@ -1,78 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from pathlib import Path - -import environ -from scancodeio.settings import * - -PROJECT_DIR = environ.Path(__file__) - 1 -ROOT_DIR = PROJECT_DIR - 1 - -# Environment - -ENV_FILE = "/etc/matchcodeio/.env" -if not Path(ENV_FILE).exists(): - ENV_FILE = ROOT_DIR(".env") - -env = environ.Env() -environ.Env.read_env(ENV_FILE) - -SECRET_KEY = env.str("SECRET_KEY") - -ALLOWED_HOSTS = env.list( - "ALLOWED_HOSTS", - default=[".localhost", "127.0.0.1", "[::1]", "host.docker.internal"], -) - -CSRF_TRUSTED_ORIGINS = env.list("CSRF_TRUSTED_ORIGINS", default=[]) - -MATCHCODEIO_REQUIRE_AUTHENTICATION = env.bool("MATCHCODEIO_REQUIRE_AUTHENTICATION", default=False) - -if not MATCHCODEIO_REQUIRE_AUTHENTICATION: - REST_FRAMEWORK["DEFAULT_PERMISSION_CLASSES"] = ("rest_framework.permissions.AllowAny",) - -INSTALLED_APPS += [ - "clearcode", - "clearindex", - "matchcode", - "minecode", - "packagedb", -] - -# Database - -DATABASES = { - "default": { - "ENGINE": env.str("SCANCODEIO_DB_ENGINE", "django.db.backends.postgresql"), - "HOST": env.str("SCANCODEIO_DB_HOST", "localhost"), - "NAME": env.str("SCANCODEIO_DB_NAME", "matchcodeio"), - "USER": env.str("SCANCODEIO_DB_USER", "matchcodeio"), - "PASSWORD": env.str("SCANCODEIO_DB_PASSWORD", "matchcodeio"), - "PORT": env.str("SCANCODEIO_DB_PORT", "5432"), - "ATOMIC_REQUESTS": True, - }, - "packagedb": { - "ENGINE": env.str("PACKAGEDB_DB_ENGINE", "django.db.backends.postgresql"), - "HOST": env.str("PACKAGEDB_DB_HOST", "localhost"), - "NAME": env.str("PACKAGEDB_DB_NAME", "packagedb"), - "USER": env.str("PACKAGEDB_DB_USER", "packagedb"), - "PASSWORD": env.str("PACKAGEDB_DB_PASSWORD", "packagedb"), - "PORT": env.str("PACKAGEDB_DB_PORT", "5432"), - "ATOMIC_REQUESTS": True, - }, -} - - -DATABASE_ROUTERS = [ - "matchcode_project.dbrouter.PackageDBRouter", - "matchcode_project.dbrouter.ScancodeIORouter", -] - -ROOT_URLCONF = "matchcode_project.urls" diff --git a/matchcode_project/urls.py b/matchcode_project/urls.py deleted file mode 100644 index 72ccd50a..00000000 --- a/matchcode_project/urls.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from django.urls import include -from django.urls import path -from django.views.generic import RedirectView - -from rest_framework import routers - -from matchcode_pipeline.api import D2DViewSet -from matchcode_pipeline.api import MatchingViewSet -from matchcode_pipeline.api import RunViewSet - -api_router = routers.DefaultRouter() -api_router.register("matching", MatchingViewSet, basename="matching") -api_router.register("d2d", D2DViewSet, basename="d2d") -api_router.register("runs", RunViewSet) - -urlpatterns = [ - path("api/", include(api_router.urls)), - path("", include("scanpipe.urls")), - path("", RedirectView.as_view(url="api/")), -] diff --git a/matchcode_project/wsgi.py b/matchcode_project/wsgi.py deleted file mode 100644 index ae478fac..00000000 --- a/matchcode_project/wsgi.py +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -""" -WSGI config for MatchCode.io. - -It exposes the WSGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/dev/howto/deployment/wsgi/ -""" - -import os - -from django.core.wsgi import get_wsgi_application - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "matchcode_project.settings") - -application = get_wsgi_application() diff --git a/minecode/tests/collectors/test_maven.py b/minecode/tests/collectors/test_maven.py index 05d31f6c..acafefd2 100644 --- a/minecode/tests/collectors/test_maven.py +++ b/minecode/tests/collectors/test_maven.py @@ -69,7 +69,7 @@ def test_map_maven_package(self): def test_map_maven_package_custom_repo_url(self): package_count = packagedb.models.Package.objects.all().count() self.assertEqual(0, package_count) - custom_repo_purl = "pkg:maven/org.eclipse.core/runtime@20070801?repository_url=https://packages.atlassian.com/mvn/maven-atlassian-external/" + custom_repo_purl = "pkg:maven/org.eclipse.core/runtime@20070801?repository_url=https://packages.atlassian.com/mvn/maven-atlassian-external" package_url = PackageURL.from_string(custom_repo_purl) maven.map_maven_package( package_url, packagedb.models.PackageContentType.BINARY, ("test_pipeline") @@ -77,7 +77,7 @@ def test_map_maven_package_custom_repo_url(self): package_count = packagedb.models.Package.objects.all().count() self.assertEqual(1, package_count) package = packagedb.models.Package.objects.all().first() - expected_repo_url = "https://packages.atlassian.com/mvn/maven-atlassian-external//org/eclipse/core/runtime/20070801/runtime-20070801.jar" + expected_repo_url = "https://packages.atlassian.com/mvn/maven-atlassian-external/org/eclipse/core/runtime/20070801/runtime-20070801.jar" self.assertEqual(expected_repo_url, package.download_url) def test_process_request(self): diff --git a/minecode/tests/testfiles/npm/expected_1000_records.json b/minecode/tests/testfiles/npm/expected_1000_records.json index 3e0100ad..04cf0626 100644 --- a/minecode/tests/testfiles/npm/expected_1000_records.json +++ b/minecode/tests/testfiles/npm/expected_1000_records.json @@ -7474,7 +7474,7 @@ { "uri":"https://registry.npmjs.org/diveSync", "source_uri":"https://replicate.npmjs.com/registry/_changes?include_docs=true&limit=1000&since=77777", - "package_url":"pkg:npm/diveSync", + "package_url":"pkg:npm/divesync", "file_name":null, "size":null, "date":null, diff --git a/minecode/tests/testfiles/npm/expected_doclimit_visitor.json b/minecode/tests/testfiles/npm/expected_doclimit_visitor.json index 18e4f398..e815a9cd 100644 --- a/minecode/tests/testfiles/npm/expected_doclimit_visitor.json +++ b/minecode/tests/testfiles/npm/expected_doclimit_visitor.json @@ -18,7 +18,7 @@ { "uri":"https://registry.npmjs.org/Reston", "source_uri":"https://replicate.npmjs.com/registry/_changes?include_docs=true&limit=1000&since=2300000", - "package_url":"pkg:npm/Reston", + "package_url":"pkg:npm/reston", "file_name":null, "size":null, "date":null, @@ -50,7 +50,7 @@ { "uri":"https://registry.npmjs.org/ClearSilver", "source_uri":"https://replicate.npmjs.com/registry/_changes?include_docs=true&limit=1000&since=2300000", - "package_url":"pkg:npm/ClearSilver", + "package_url":"pkg:npm/clearsilver", "file_name":null, "size":null, "date":null, @@ -66,7 +66,7 @@ { "uri":"https://registry.npmjs.org/FreshDocs", "source_uri":"https://replicate.npmjs.com/registry/_changes?include_docs=true&limit=1000&since=2300000", - "package_url":"pkg:npm/FreshDocs", + "package_url":"pkg:npm/freshdocs", "file_name":null, "size":null, "date":null, @@ -82,7 +82,7 @@ { "uri":"https://registry.npmjs.org/Journaling-Hash", "source_uri":"https://replicate.npmjs.com/registry/_changes?include_docs=true&limit=1000&since=2300000", - "package_url":"pkg:npm/Journaling-Hash", + "package_url":"pkg:npm/journaling-hash", "file_name":null, "size":null, "date":null, @@ -738,7 +738,7 @@ { "uri":"https://registry.npmjs.org/NormAndVal", "source_uri":"https://replicate.npmjs.com/registry/_changes?include_docs=true&limit=1000&since=2300000", - "package_url":"pkg:npm/NormAndVal", + "package_url":"pkg:npm/normandval", "file_name":null, "size":null, "date":null, @@ -754,7 +754,7 @@ { "uri":"https://registry.npmjs.org/OnCollect", "source_uri":"https://replicate.npmjs.com/registry/_changes?include_docs=true&limit=1000&since=2300000", - "package_url":"pkg:npm/OnCollect", + "package_url":"pkg:npm/oncollect", "file_name":null, "size":null, "date":null, @@ -994,7 +994,7 @@ { "uri":"https://registry.npmjs.org/Birbal", "source_uri":"https://replicate.npmjs.com/registry/_changes?include_docs=true&limit=1000&since=2300000", - "package_url":"pkg:npm/Birbal", + "package_url":"pkg:npm/birbal", "file_name":null, "size":null, "date":null, @@ -1282,7 +1282,7 @@ { "uri":"https://registry.npmjs.org/Cashew", "source_uri":"https://replicate.npmjs.com/registry/_changes?include_docs=true&limit=1000&since=2300000", - "package_url":"pkg:npm/Cashew", + "package_url":"pkg:npm/cashew", "file_name":null, "size":null, "date":null, diff --git a/packagedb/management/commands/subscribe_package.py b/packagedb/management/commands/subscribe_package.py index 6651ff27..3b78fe25 100644 --- a/packagedb/management/commands/subscribe_package.py +++ b/packagedb/management/commands/subscribe_package.py @@ -12,7 +12,7 @@ from aboutcode.federatedcode.client import subscribe_package -from purldb_project import settings +from purldb import settings class Command(BaseCommand): diff --git a/packagedb/sbom.py b/packagedb/sbom.py index 1a49dfec..6bc1c9a8 100644 --- a/packagedb/sbom.py +++ b/packagedb/sbom.py @@ -29,7 +29,7 @@ from cyclonedx.schema import SchemaVersion from cyclonedx.validation.json import JsonStrictValidator -from purldb_project import __version__ as purldb_version +from purldb import __version__ as purldb_version def get_cyclonedx_bom(package): diff --git a/packagedb/tests/testfiles/api/package-filter_by_checksums-enhanced-package-data-expected.json b/packagedb/tests/testfiles/api/package-filter_by_checksums-enhanced-package-data-expected.json index 8860e627..6293edf1 100644 --- a/packagedb/tests/testfiles/api/package-filter_by_checksums-enhanced-package-data-expected.json +++ b/packagedb/tests/testfiles/api/package-filter_by_checksums-enhanced-package-data-expected.json @@ -96,9 +96,9 @@ "source_packages":[], "extra_data":{}, "dependencies":[], - "package_uid":"pkg:npm/example/Bar@56.78?uuid=fixed-uid-done-for-testing-5642512d1758", + "package_uid":"pkg:npm/example/bar@56.78?uuid=fixed-uid-done-for-testing-5642512d1758", "datasource_id":null, - "purl":"pkg:npm/example/Bar@56.78", + "purl":"pkg:npm/example/bar@56.78", "repository_homepage_url":null, "repository_download_url":null, "api_data_url":null, diff --git a/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json b/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json index 1ed603e2..1b2b5a3d 100644 --- a/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json +++ b/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json @@ -64,7 +64,7 @@ { "filename":"Bar.zip", "package_content":null, - "purl":"pkg:npm/example/Bar@56.78", + "purl":"pkg:npm/example/bar@56.78", "type":"npm", "namespace":"example", "name":"Bar", @@ -101,7 +101,7 @@ "notice_text":null, "source_packages":[], "extra_data":{}, - "package_uid":"pkg:npm/example/Bar@56.78?uuid=fixed-uid-done-for-testing-5642512d1758", + "package_uid":"pkg:npm/example/bar@56.78?uuid=fixed-uid-done-for-testing-5642512d1758", "datasource_id":null, "file_references":[], "dependencies":[] diff --git a/packagedb/tests/testfiles/sbom/package-sbom-expected.json b/packagedb/tests/testfiles/sbom/package-sbom-expected.json index 9d018e39..7ebc9f4c 100644 --- a/packagedb/tests/testfiles/sbom/package-sbom-expected.json +++ b/packagedb/tests/testfiles/sbom/package-sbom-expected.json @@ -54,13 +54,13 @@ } ], "dependencies":[ - {}, - {}, { "dependsOn":[ "pkg:generic/dep1", "pkg:generic/dep2" ] - } + }, + {}, + {} ] } \ No newline at end of file diff --git a/purldb_project/__init__.py b/purldb/__init__.py similarity index 87% rename from purldb_project/__init__.py rename to purldb/__init__.py index 7be6bfe2..7d452648 100644 --- a/purldb_project/__init__.py +++ b/purldb/__init__.py @@ -17,5 +17,5 @@ def command_line(): """Command line entry point.""" from django.core.management import execute_from_command_line - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "purldb_project.settings") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "purldb.settings") execute_from_command_line(sys.argv) diff --git a/purldb_project/settings.py b/purldb/settings.py similarity index 98% rename from purldb_project/settings.py rename to purldb/settings.py index c6c2c731..ed2c7ffb 100644 --- a/purldb_project/settings.py +++ b/purldb/settings.py @@ -12,7 +12,7 @@ import environ -from purldb_project import __version__ +from purldb import __version__ PURLDB_VERSION = __version__ @@ -73,12 +73,14 @@ "django.contrib.staticfiles", "django.contrib.admin", "django.contrib.humanize", + "django.contrib.postgres", # Third-party apps "django_filters", "rest_framework", "drf_spectacular", "rest_framework.authtoken", "django_rq", + "scanpipe", ) MIDDLEWARE = ( @@ -91,9 +93,9 @@ "django.middleware.clickjacking.XFrameOptionsMiddleware", ) -ROOT_URLCONF = "purldb_project.urls" +ROOT_URLCONF = "purldb.urls" -WSGI_APPLICATION = "purldb_project.wsgi.application" +WSGI_APPLICATION = "purldb.wsgi.application" SECURE_PROXY_SSL_HEADER = env.tuple( "SECURE_PROXY_SSL_HEADER", default=("HTTP_X_FORWARDED_PROTO", "https") diff --git a/purldb_project/static/.keep b/purldb/static/.keep similarity index 100% rename from purldb_project/static/.keep rename to purldb/static/.keep diff --git a/purldb_project/urls.py b/purldb/urls.py similarity index 92% rename from purldb_project/urls.py rename to purldb/urls.py index 73bce815..602c2ce4 100644 --- a/purldb_project/urls.py +++ b/purldb/urls.py @@ -19,6 +19,9 @@ from matchcode.api import ApproximateDirectoryContentIndexViewSet from matchcode.api import ApproximateDirectoryStructureIndexViewSet +from matchcode.api import D2DViewSet +from matchcode.api import MatchingViewSet +from matchcode.api import RunViewSet from minecode.api import ScannableURIViewSet from minecode.api import index_package_scan from packagedb.api import CollectViewSet @@ -47,7 +50,9 @@ "approximate_directory_structure_index", ApproximateDirectoryStructureIndexViewSet ) api_router.register("package_activity", PackageActivityViewSet) - +api_router.register("matching", MatchingViewSet, basename="matching") +api_router.register("d2d", D2DViewSet, basename="d2d") +api_router.register("runs", RunViewSet) urlpatterns = [ path( diff --git a/purldb_project/wsgi.py b/purldb/wsgi.py similarity index 88% rename from purldb_project/wsgi.py rename to purldb/wsgi.py index bfb452a3..6b290d6f 100644 --- a/purldb_project/wsgi.py +++ b/purldb/wsgi.py @@ -18,6 +18,6 @@ """ -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "purldb_project.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "purldb.settings") application = get_wsgi_application() diff --git a/purldb_public_project/__init__.py b/purldb_public_project/__init__.py deleted file mode 100644 index a77c6343..00000000 --- a/purldb_public_project/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# PurlDB is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import os -import sys - - -def command_line(): - """Command line entry point.""" - from django.core.management import execute_from_command_line - - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "purldb_public_project.settings") - execute_from_command_line(sys.argv) diff --git a/purldb_public_project/settings.py b/purldb_public_project/settings.py deleted file mode 100644 index f990759e..00000000 --- a/purldb_public_project/settings.py +++ /dev/null @@ -1,14 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from purldb_project.settings import * - -ROOT_URLCONF = "purldb_public_project.urls" - -WSGI_APPLICATION = "purldb_public_project.wsgi.application" diff --git a/purldb_public_project/static/.keep b/purldb_public_project/static/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/purldb_public_project/urls.py b/purldb_public_project/urls.py deleted file mode 100644 index fddb594b..00000000 --- a/purldb_public_project/urls.py +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from django.conf.urls import include -from django.urls import path -from django.views.generic import RedirectView -from django.views.generic.base import TemplateView - -from drf_spectacular.views import SpectacularAPIView -from drf_spectacular.views import SpectacularSwaggerView -from rest_framework import routers - -from packagedb.api import PackagePublicViewSet -from packagedb.api import PurlValidateViewSet -from packagedb.api import ResourceViewSet - -api_router = routers.DefaultRouter() -api_router.register("packages", PackagePublicViewSet) -api_router.register("resources", ResourceViewSet) -api_router.register("validate", PurlValidateViewSet, "validate") - - -urlpatterns = [ - path( - "robots.txt", - TemplateView.as_view(template_name="robots.txt", content_type="text/plain"), - ), - path("api/", include((api_router.urls, "api"))), - path("", RedirectView.as_view(url="api/")), - path("api/schema/", SpectacularAPIView.as_view(), name="schema"), - path( - "api/docs/", - SpectacularSwaggerView.as_view(url_name="schema"), - name="swagger-ui", - ), -] diff --git a/purldb_public_project/wsgi.py b/purldb_public_project/wsgi.py deleted file mode 100644 index 242c50a8..00000000 --- a/purldb_public_project/wsgi.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import os - -from django.core.wsgi import get_wsgi_application - -""" -WSGI config for purldb-public. - -It exposes the WSGI callable as a module-level variable named ``application``. -""" - - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "purldb_public_project.settings") - -application = get_wsgi_application() diff --git a/pyproject.toml b/pyproject.toml index 2adb8898..ef846232 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -144,8 +144,6 @@ max-complexity = 10 "tests/*" = ["S101"] # Use of assert detected "test_*.py" = ["S101"] # Use of assert detected "**/testfiles/**.py" = ["F821"] # Ignore undefined names from test files -"matchcode_project/settings.py" = ["F403", "F405"] # Ignore undefined names from star imports and star imports -"purldb_public_project/settings.py" = ["F403", "F405"] # Ignore undefined names from star imports and star imports "etc/scripts/update_skeleton.py" = ["S108", "S603", "S605", "S607"] "purl2vcs/src/purl2vcs/find_source_repo.py" = ["S605"] diff --git a/requirements-dev.txt b/requirements-dev.txt index 13e9f44e..020000df 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,53 +1,49 @@ alabaster==1.0.0 -anyio==4.12.0 -babel==2.17.0 -backports.tarfile==1.2.0 -black==25.12.0 -bracex==2.6 -bump-my-version==1.2.5 -django-debug-toolbar==6.1.0 +anyio==4.14.1 +babel==2.18.0 +black==26.5.1 +bracex==3.0 +bump-my-version==1.4.1 doc8==2.0.0 docutils==0.21.2 -exceptiongroup==1.3.1 execnet==2.1.2 h11==0.16.0 -httpcore==1.0.9 -httpx==0.28.1 -id==1.5.0 -imagesize==1.4.1 +httpcore2==2.5.0 +httpx2==2.5.0 +id==1.6.1 +imagesize==2.0.0 iniconfig==2.3.0 -isort==7.0.0 +isort==8.0.1 jaraco.classes==3.4.0 -jaraco.context==6.0.1 +jaraco.context==6.1.2 jeepney==0.9.0 keyring==25.7.0 mypy_extensions==1.1.0 -nh3==0.3.2 -pathspec==0.12.1 -platformdirs==4.5.1 +nh3==0.3.6 +pathspec==1.1.1 prompt_toolkit==3.0.52 pycodestyle==2.14.0 -pytest==9.0.2 -pytest-django==4.11.1 -pytest-rerunfailures==16.1 +pytest==9.1.1 +pytest-django==4.12.0 +pytest-rerunfailures==16.4 pytest-xdist==3.8.0 -pytokens==0.3.0 +pytokens==0.4.1 questionary==2.1.1 -readme_renderer==44.0 +readme_renderer==45.0 requests-toolbelt==1.0.0 restructuredtext_lint==2.0.2 rfc3986==2.0.0 -rich==14.2.0 -rich-click==1.9.4 -ruff==0.14.9 +rich-click==1.9.8 +roman-numerals==4.1.0 +ruff==0.15.21 secretstorage==3.5.0 -snowballstemmer==3.0.1 -sphinx==8.1.3 -sphinx-autobuild==2024.10.3 +snowballstemmer==3.1.1 +sphinx==9.1.0 +sphinx-autobuild==2025.8.25 sphinx-copybutton==0.5.2 +sphinx-reredirects==1.1.0 sphinx-rtd-dark-mode==1.3.0 -sphinx-rtd-theme==3.0.2 -sphinx_reredirects==0.1.6 +sphinx_rtd_theme==3.1.0 sphinxcontrib-applehelp==2.0.0 sphinxcontrib-devhelp==2.0.0 sphinxcontrib-htmlhelp==2.1.0 @@ -55,11 +51,12 @@ sphinxcontrib-jquery==4.1 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==2.0.0 sphinxcontrib-serializinghtml==2.0.0 -starlette==0.50.0 -stevedore==5.6.0 -tomlkit==0.13.3 +starlette==1.3.1 +stevedore==5.9.0 +tomlkit==0.15.0 +truststore==0.10.4 twine==6.2.0 -uvicorn==0.38.0 -watchfiles==1.1.1 -wcmatch==10.1 -websockets==15.0.1 \ No newline at end of file +uvicorn==0.51.0 +watchfiles==1.2.0 +wcmatch==11.0 +websockets==16.1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 57c86ef4..58d24ed1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,181 +1,182 @@ aboutcode-toolkit==11.1.1 -aboutcode.federated==0.1.0 +aboutcode.api_auth==0.2.0 +aboutcode.federated==1.0.4 aboutcode.federatedcode==0.1.0 aboutcode.hashid==0.2.0 aboutcode.pipeline==0.2.1 aiofiles==25.1.0 -aiohappyeyeballs==2.6.1 -aiohttp==3.13.2 +aiohappyeyeballs==2.7.1 +aiohttp==3.14.1 aiosignal==1.4.0 annotated-types==0.7.0 arrow==1.4.0 -asgiref==3.11.0 -async-timeout==5.0.1 -attrs==25.4.0 -banal==1.0.6 +asgiref==3.11.1 +attrs==26.1.0 +banal==1.1.2 beartype==0.22.9 -beautifulsoup4==4.13.4 -bids-analyser==0.2.0 -binary-inspector==0.1.2 -binaryornot==0.4.4 -bitarray==3.8.0 -bleach==6.2.0 +beautifulsoup4==4.14.3 +bids-analyser==0.3.1 +binary-inspector==0.2.0 +binaryornot==0.6.0 +bitarray==3.9.0 +bleach==6.4.0 boolean.py==5.0 -certifi==2025.11.12 -cffi==2.0.0 -chardet==5.2.0 -charset-normalizer==3.4.4 +certifi==2026.6.17 +cffi==2.1.0 +chardet==7.4.3 +charset-normalizer==3.4.7 clamd==1.0.2 -click==8.2.1 +click==8.4.2 colorama==0.4.6 -commoncode==32.3.0 +commoncode==32.5.2 construct==2.10.70 -container-inspector==33.0.0 +container-inspector==33.1.0 crispy-bootstrap3==2024.1 -croniter==6.0.0 +croniter==6.2.3 crontab==1.0.5 -cryptography==46.0.3 -cyclonedx-python-lib==11.2.0 +cryptography==49.0.0 +cyclonedx-python-lib==11.11.0 +cyseq==0.1.3 debian_inspector==31.1.1 defusedxml==0.7.1 -Django==5.1.13 -django-crispy-forms==2.4 -django-environ==0.12.0 -django-filter==25.1 -django-htmx==1.26.0 -django-probes==1.7.0 -django-rq==3.1 -django-taggit==6.1.0 -djangorestframework==3.16.1 +Django==6.0.6 +django-crispy-forms==2.6 +django-debug-toolbar==7.0.0 +django-environ==0.14.0 +django-filter==25.2 +django-htmx==1.27.0 +django-probes==1.8.0 +django-rq==4.1.0 +djangorestframework==3.17.1 dockerfile-parse==2.0.1 dparse2==0.7.0 -drf-spectacular==0.29.0 -elementpath==5.0.4 +drf-spectacular==0.30.0 +elementpath==5.1.3 elf_inspector==0.0.3 et_xmlfile==2.0.0 -extractcode==31.0.0 +extractcode==31.1.0 extractcode-7z==16.5.210531 extractcode-libarchive==3.5.1.210531 fasteners==0.20 fastjsonschema==2.21.2 -fetchcode==0.8.0 +fetchcode==0.8.2 fetchcode-container==1.2.3.210512 fingerprints==1.2.3 -flot==0.7.2 +flot==0.7.3 freezegun==1.5.5 frozenlist==1.8.0 ftfy==6.3.1 -ftputil==5.1.0 +ftputil==5.2.0 fusepy==3.0.1 gemfileparser2==0.9.4 gitdb==4.0.12 -GitPython==3.1.45 +GitPython==3.1.50 go-inspector==0.5.0 -gunicorn==23.0.0 +gunicorn==26.0.0 html5lib==1.1 htmllistparse==0.6.1 -idna==3.11 -importlib_metadata==8.7.0 +idna==3.18 +importlib_metadata==8.7.1 inflection==0.5.1 -intbitset==4.0.0 -isodate==0.7.2 -jaraco.functools==4.3.0 +intbitset==4.1.2 +jaraco.functools==4.5.0 javaproperties==0.8.2 jawa==2.2.0 Jinja2==3.1.6 -jsonschema==4.25.1 +jsonschema==4.26.0 jsonschema-specifications==2025.9.1 jsonstreams==0.6.0 -lib4sbom==0.9.1 +lib4sbom==0.10.4 license-expression==30.4.4 -lief==0.15.1 -lxml==6.0.2 -Markdown==3.10 +lief==0.17.0 +linkify-it-py==2.1.0 +lxml==6.1.1 markdown-it-py==4.0.0 MarkupSafe==3.0.3 matchcode-toolkit==7.2.2 +mdit-py-plugins==0.6.1 mdurl==0.1.2 milksnake==0.1.6 -minecode_pipelines==0.1.1 +minecode_pipelines==1.0.1 mock==5.2.0 -more-itertools==10.8.0 -multidict==6.7.0 +more-itertools==11.1.0 +multidict==6.7.1 +multiregex==2.0.4 natsort==8.4.0 normality==2.6.1 -numpy==2.2.6 openpyxl==3.1.5 packagedcode-msitools==0.101.210706 -packageurl-python==0.17.5 -packaging==25.0 +packageurl-python==0.17.6 +packaging==26.2 packvers==21.5 parameter-expansion-patched==0.3.1 -pdfminer.six==20251107 +pdfminer.six==20260107 pefile==2024.8.26 -pip==25.3 +pip==26.1.2 pip-requirements-parser==32.0.1 pkginfo2==30.1.0 +platformdirs==4.10.0 pluggy==1.6.0 plugincode==32.0.0 ply==3.11 -propcache==0.4.1 -psycopg==3.2.10 -psycopg-binary==3.2.10 +propcache==0.5.2 +psycopg==3.3.4 +psycopg-binary==3.3.4 publicsuffix2==2.20191221 py-serializable==2.1.0 -py-tree-sitter-swift==0.0.1 -pyahocorasick==2.2.0 -pycparser==2.23 -pydantic==2.12.5 -pydantic-settings==2.12.0 -pydantic_core==2.41.5 -pyelftools==0.32 -PyGithub==2.8.1 +pyahocorasick==2.3.1 +pycparser==3.0 +pydantic==2.13.4 +pydantic-settings==2.14.2 +pydantic_core==2.46.4 +pyelftools==0.33 +PyGithub==2.9.1 pygmars==1.0.0 -Pygments==2.19.2 -pyinstrument==5.1.1 -PyJWT==2.10.1 +Pygments==2.20.0 +pyinstrument==5.1.2 +PyJWT==2.13.0 pymaven-patch==0.3.2 -PyNaCl==1.6.1 -pyparsing==3.2.5 +PyNaCl==1.6.2 +pyparsing==3.3.2 python-dateutil==2.9.0.post0 -python-dotenv==1.2.1 -python-inspector==0.14.4 -pytz==2025.2 +python-dotenv==1.2.2 +python-inspector==0.15.2 +pytz==2026.2 PyYAML==6.0.3 -rdflib==7.5.0 -redis==6.4.0 +rdflib==7.6.0 +redis==8.0.1 referencing==0.37.0 -regipy==5.2.0 +regipy==6.2.1 reppy2==0.3.6 -requests==2.32.5 +requests==2.34.2 resolvelib==1.2.1 -rpds-py==0.30.0 +rich==15.0.0 +rpds-py==2026.6.3 rpm-inspector-rpm==4.16.1.3.210404 -rq==2.6.0 +rq==2.10.0 rq-scheduler==0.14.0 rubymarshal==1.0.3 -rust-inspector==0.1.0 +rust-inspector==0.2.1 samecode==0.5.1 saneyaml==0.6.1 -scancode-toolkit==32.4.0 -scancodeio==35.5.0 -scipy==1.15.3 +scancode-toolkit==32.5.0 +scancodeio==38.0.0 scorecode==0.0.4 semantic-version==2.10.0 semver==3.0.4 -setuptools==80.9.0 +setuptools==82.0.1 six==1.17.0 -smmap==5.0.2 +smmap==5.0.3 sortedcontainers==2.4.0 -soupsieve==2.8 -source-inspector==0.7.0 +soupsieve==2.8.4 +source-inspector==0.7.1 spdx-tools==0.8.2 -sqlparse==0.5.4 +sqlparse==0.5.5 symbolic==10.2.1 -tantivy==0.25.1 +tantivy==0.26.0 text-unidecode==1.3 +textual==8.2.8 toml==0.10.2 -tomli==2.3.0 tree-sitter==0.23.2 tree-sitter-bash==0.21.0 tree-sitter-c==0.21.1 @@ -187,21 +188,23 @@ tree-sitter-javascript==0.21.2 tree-sitter-objc==3.0.2 tree-sitter-python==0.21.0 tree-sitter-rust==0.21.2 -typecode==30.0.2 +tree-sitter-swift==0.0.1 +typecode==30.2.0 typecode-libmagic==5.39.210531 typecode_libmagic_system_provided==33.0.0 typing-inspection==0.4.2 -typing_extensions==4.15.0 -tzdata==2025.3 -univers==31.1.0 +typing_extensions==4.16.0 +tzdata==2026.3 +uc-micro-py==2.0.0 +univers==32.0.1 uritemplate==4.2.0 -uritools==5.0.0 -urllib3==2.6.2 +uritools==6.1.2 +urllib3==2.7.0 urlpy==0.5 -wcwidth==0.2.14 +wcwidth==0.8.2 webencodings==0.5.1 xlsxwriter==3.2.9 -xmlschema==4.2.0 -xmltodict==1.0.2 -yarl==1.22.0 -zipp==3.23.0 +xmlschema==4.3.2 +xmltodict==1.0.4 +yarl==1.24.2 +zipp==4.1.0 diff --git a/setup.cfg b/setup.cfg index 827ff26c..310b42d2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,40 +40,25 @@ include_package_data = true zip_safe = false python_requires = >=3.10 install_requires = - aboutcode.pipeline >= 0.2.1 - arrow >= 1.3.0 - debian-inspector >= 31.1.1 - commoncode >= 32.3.0 - Django >= 5.1.11 - django-environ >= 0.12.0 - django-rq >= 3.0.1 - djangorestframework >= 3.16.0 - django-filter >= 25.1 - drf-spectacular >= 0.28.0 - fetchcode >= 0.6.0 - gunicorn >= 23.0.0 - ftputil >= 5.1.0 - jawa >= 2.2.0 - markdown >= 3.8.2 - natsort >= 8.4.0 - packageurl-python >= 0.17.3 - psycopg[binary] >= 3.2.9 - PyGithub >= 2.6.1 - reppy2 >= 0.3.6 - rq-scheduler >= 0.14.0 - rubymarshal == 1.0.3 - scancode-toolkit[packages] >= 32.4.0 - urlpy >= 0.5 - matchcode-toolkit >= 7.2.2 - purl2vcs >= 2.0.0 - univers >= 31.0.0 - scancodeio == 35.5.0 - GitPython >= 3.1.44 - samecode >= 0.5.1 - # FederatedCode integration - aboutcode.federatedcode >= 0.1.0 - aboutcode.federated >= 0.1.0 + scancodeio == 38.0.0 + PyGithub == 2.9.1 + aboutcode.federated == 1.0.4 + aboutcode.federatedcode == 0.1.0 + arrow == 1.4.0 + debian-inspector == 31.1.1 + drf-spectacular == 0.30.0 + fetchcode == 0.8.2 + ftputil == 5.2.0 + jawa == 2.2.0 + matchcode-toolkit == 7.2.2 minecode-pipelines + natsort == 8.4.0 + purl2vcs == 2.0.0 + reppy2 == 0.3.6 + rq-scheduler == 0.14.0 + rubymarshal == 1.0.3 + samecode == 0.5.1 + urlpy == 0.5 setup_requires = setuptools_scm[toml] >= 4 @@ -102,8 +87,8 @@ dev = [options.entry_points] console_scripts = - purldb = purldb_project:command_line + purldb = purldb:command_line scancodeio_pipelines = - matching = matchcode_pipeline.pipelines.matching:Matching + matching = matchcode.pipelines.matching:Matching d2d = scanpipe.pipelines.deploy_to_develop:DeployToDevelop