Merge matchcode purldb

Makefile

@@ -10,8 +10,7 @@
 # Python version can be specified with `$ PYTHON_EXE=python3.x make conf`
 PYTHON_EXE?=python3
 VENV=venv
-MANAGE=${VENV}/bin/python manage_purldb.py
-MATCHCODE_MANAGE=${VENV}/bin/python manage_matchcode.py
+MANAGE=${VENV}/bin/python manage.py
 ACTIVATE?=. ${VENV}/bin/activate;
 VIRTUALENV_PYZ=../etc/thirdparty/virtualenv.pyz
 # Do not depend on Python to generate the SECRET_KEY
@@ -22,12 +21,9 @@ ENV_FILE=.env
 
 # Customize with `$ make postgres PACKAGEDB_DB_PASSWORD=YOUR_PASSWORD`
 PACKAGEDB_DB_PASSWORD=packagedb
-MATCHCODEIO_DB_PASSWORD=matchcodeio
-SCANCODEIO_DB_PASSWORD=scancodeio
 
 # Django settings shortcuts
-DJSM_PDB=DJANGO_SETTINGS_MODULE=purldb_project.settings
-DJSM_MAT=DJANGO_SETTINGS_MODULE=matchcode_project.settings
+DJSM_PDB=DJANGO_SETTINGS_MODULE=purldb.settings
 
 # Use sudo for postgres, but only on Linux
 UNAME := $(shell uname)
@@ -58,8 +54,6 @@ envfile:
 envfile_testing: envfile
 	@echo PACKAGEDB_DB_USER=\"postgres\" >> ${ENV_FILE}
 	@echo PACKAGEDB_DB_PASSWORD=\"postgres\" >> ${ENV_FILE}
-	@echo SCANCODEIO_DB_USER=\"postgres\" >> ${ENV_FILE}
-	@echo SCANCODEIO_DB_PASSWORD=\"postgres\" >> ${ENV_FILE}
 
 doc8:
 	@echo "-> Run doc8 validation"
@@ -101,23 +95,9 @@ postgres:
 	${SUDO_POSTGRES} createdb --encoding=utf-8 --owner=packagedb packagedb
 	@$(MAKE) migrate
 
-postgres_matchcodeio:
-	@echo "-> Configure PostgreSQL database"
-	@echo "-> Create database user 'matchcodeio'"
-	${SUDO_POSTGRES} createuser --no-createrole --no-superuser --login --inherit --createdb matchcodeio || true
-	${SUDO_POSTGRES} psql -c "alter user matchcodeio with encrypted password '${MATCHCODEIO_DB_PASSWORD}';" || true
-	@echo "-> Drop 'matchcodeio' database"
-	${SUDO_POSTGRES} dropdb matchcodeio || true
-	@echo "-> Create 'matchcodeio' database"
-	${SUDO_POSTGRES} createdb --encoding=utf-8 --owner=matchcodeio matchcodeio
-	${MATCHCODE_MANAGE} migrate
-
 run:
 	${MANAGE} runserver 8001 --insecure
 
-run_matchcodeio:
-	${MATCHCODE_MANAGE} runserver 8002 --insecure
-
 seed:
 	${MANAGE} seed
 
@@ -128,20 +108,14 @@ run_map:
 	${MANAGE} run_map
 
 test_purldb:
-	${ACTIVATE} ${DJSM_PDB} pytest -vvs --lf minecode packagedb purl2vcs purldb_project purldb_public_project --ignore packagedb/tests/test_throttling.py
+	${ACTIVATE} ${DJSM_PDB} pytest -vvs --lf minecode matchcode packagedb purl2vcs purldb clearcode clearindex --ignore packagedb/tests/test_throttling.py
 	${ACTIVATE} ${DJSM_PDB} pytest -vvs --lf packagedb/tests/test_throttling.py
 
-test_clearcode:
-	${ACTIVATE} ${DJSM_PDB} ${PYTHON_EXE} -m pytest -vvs clearcode clearindex
-
-test_matchcode:
-	${ACTIVATE} ${DJSM_MAT} ${PYTHON_EXE} -m pytest -vvs matchcode_pipeline matchcode
-
-test: test_purldb test_matchcode test_toolkit test_clearcode
-
-test_minecode:
+test_minecode_pipelines:
 	${ACTIVATE} ${PYTHON_EXE} -m pytest -vvs minecode_pipelines
 
+test: test_purldb test_minecode_pipelines
+
 shell:
 	${MANAGE} shell
 
@@ -179,4 +153,4 @@ docker-images:
 	@docker save minecode minecode_minecode nginx | gzip > dist/minecode-images-`git describe --tags`.tar.gz
 
 # keep this sorted
-.PHONY: bump check docs-check clean clearindex clearsync conf dev docker-images docs envfile envfile_testing index_packages migrate postgres postgres_matchcodeio priority_queue run run_map run_matchcodeio run_visit seed shell test test_clearcode test_matchcode test_purldb test_toolkit valid virtualenv
+.PHONY: bump check docs-check clean clearindex clearsync conf dev docker-images docs envfile envfile_testing index_packages migrate postgres priority_queue run run_map run_visit seed shell test test_purldb test_minecode_pipelines valid virtualenv

docker-compose.common.yml

@@ -18,9 +18,9 @@ services:
   web:
     build: .
     command: wait-for-it --strict --timeout=60 db:5432 -- sh -c "
-        python manage_purldb.py migrate &&
-        python manage_purldb.py collectstatic --no-input --verbosity 0 --clear &&
-        gunicorn purldb_project.wsgi:application --bind :8000 --timeout 600 --workers 8"
+        python manage.py migrate &&
+        python manage.py collectstatic --no-input --verbosity 0 --clear &&
+        gunicorn purldb.wsgi:application --bind :8000 --timeout 600 --workers 8"
     env_file:
       - docker_purldb.env
     expose:
@@ -35,7 +35,7 @@ services:
 
   priority_queue:
     build: .
-    command: wait-for-it web:8000 -- python manage_purldb.py priority_queue
+    command: wait-for-it web:8000 -- python manage.py priority_queue
     env_file:
       - docker_purldb.env
     volumes:
@@ -49,7 +49,7 @@ services:
 
   scheduler:
     build: .
-    command: wait-for-it web:8000 -- python manage_purldb.py run_scheduler
+    command: wait-for-it web:8000 -- python manage.py run_scheduler
     env_file:
       - docker_purldb.env
     volumes:
@@ -62,7 +62,7 @@ services:
 
   rq_worker:
     build: .
-    command: wait-for-it web:8000 -- python manage_purldb.py rqworker default
+    command: wait-for-it web:8000 -- python manage.py rqworker default
     env_file:
       - docker_purldb.env
     volumes:

docker-compose.miners.yml

@@ -5,8 +5,8 @@ services:
   visitor:
     build: .
     command: sh -c "
-        wait-for-it web:8000 -- python manage_purldb.py seed &&
-        python manage_purldb.py run_visit --ignore-robots --ignore-throttle"
+        wait-for-it web:8000 -- python manage.py seed &&
+        python manage.py run_visit --ignore-robots --ignore-throttle"
     env_file:
       - docker_purldb.env
     volumes:
@@ -19,7 +19,7 @@ services:
 
   mapper:
     build: .
-    command: wait-for-it web:8000 -- python manage_purldb.py run_map
+    command: wait-for-it web:8000 -- python manage.py run_map
     env_file:
       - docker_purldb.env
     volumes:
@@ -45,7 +45,7 @@ services:
 
   clearindex:
     build: .
-    command: wait-for-it web:8000 -- python manage_purldb.py run_clearindex
+    command: wait-for-it web:8000 -- python manage.py run_clearindex
     env_file:
       - docker_purldb.env
     profiles:

docs/source/conf.py

@@ -12,7 +12,7 @@
 import os
 import sys
 
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "purldb_project.settings")
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "purldb.settings")
 sys.path.insert(0, os.path.abspath("../../."))
 
 # -- Project information -----------------------------------------------------

docs/source/how-to-guides/purl_watch_how_to.rst

@@ -15,7 +15,7 @@ following management command on a periodic basis:
 
    .. code-block:: bash
 
-        python manage_purldb.py watch_packages
+        python manage.py watch_packages
 
    This command checks all PURLs in your PurlDB for new versions and indexes them accordingly.
 

docs/source/purldb/purl_watch.rst

@@ -16,7 +16,7 @@ Using the Management Command
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 For a relatively small and focused PurlDB, one can use the management
-command ``python manage_purldb.py watch_packages``. This command can be
+command ``python manage.py watch_packages``. This command can be
 run periodically using a cron job to watch all the PURLs in your PurlDB for
 new versions. Upon detecting new versions, it collects and indexes the new package.
 This approach is efficient for smaller databases.

manage_purldb.py → manage.py

@@ -14,5 +14,5 @@
 if __name__ == "__main__":
     from django.core.management import execute_from_command_line
 
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "purldb_project.settings")
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "purldb.settings")
     execute_from_command_line(sys.argv)