diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index ab8ebb3dce0..0b8a1a16a4d 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -1,3 +1,4 @@ +SECRET_KEY = "{{ django_secret }}" CONTENT_ORIGIN = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}" ANSIBLE_API_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}" ANSIBLE_CONTENT_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}/pulp/content" diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index e0891b7ab5d..4ef94c861cf 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -18,6 +18,8 @@ ansible.builtin.template: src: "settings.py.j2" dest: "settings/settings.py" + vars: + django_secret: "lookup('community.general.random_string', length=50, overwrite_all='abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)')" - name: "Setup docker networking" community.docker.docker_network: diff --git a/CHANGES/+django_secret.bugfix b/CHANGES/+django_secret.bugfix new file mode 100644 index 00000000000..571ee9046f7 --- /dev/null +++ b/CHANGES/+django_secret.bugfix @@ -0,0 +1 @@ +Stopped shipping an insecure default as DJANGO_SECRET. diff --git a/pulp_file/tests/functional/api/test_domains.py b/pulp_file/tests/functional/api/test_domains.py index 51869b8114e..c88eecc0048 100644 --- a/pulp_file/tests/functional/api/test_domains.py +++ b/pulp_file/tests/functional/api/test_domains.py @@ -2,8 +2,8 @@ import uuid import pytest +from django.conf import settings -from pulpcore.app import settings from pulpcore.tests.functional.utils import download_file, generate_iso pytestmark = pytest.mark.skipif(not settings.DOMAIN_ENABLED, reason="Domains not enabled.") diff --git a/pulp_file/tests/functional/api/test_pulp_export.py b/pulp_file/tests/functional/api/test_pulp_export.py index 4650efc0498..d1da49f6b9f 100644 --- a/pulp_file/tests/functional/api/test_pulp_export.py +++ b/pulp_file/tests/functional/api/test_pulp_export.py @@ -1,8 +1,8 @@ import uuid import pytest +from django.conf import settings -from pulpcore.app import settings from pulpcore.client.pulpcore.exceptions import ApiException from pulpcore.constants import TASK_STATES diff --git a/pulpcore/app/access_policy.py b/pulpcore/app/access_policy.py index 6cb41c02b7b..6654f5259c8 100644 --- a/pulpcore/app/access_policy.py +++ b/pulpcore/app/access_policy.py @@ -1,9 +1,9 @@ from copy import deepcopy +from django.conf import settings from rest_access_policy import AccessPolicy from rest_framework.exceptions import APIException -from pulpcore.app import settings from pulpcore.app.models import AccessPolicy as AccessPolicyModel from pulpcore.app.util import get_view_urlpattern, get_viewset_for_model diff --git a/pulpcore/app/authentication.py b/pulpcore/app/authentication.py index 0e42ec0871d..745e35e1ffa 100644 --- a/pulpcore/app/authentication.py +++ b/pulpcore/app/authentication.py @@ -5,6 +5,7 @@ from gettext import gettext as _ import jq +from django.conf import settings from django.contrib.auth import authenticate from django.contrib.auth.backends import RemoteUserBackend from rest_framework.authentication import ( @@ -16,8 +17,6 @@ ) from rest_framework.exceptions import AuthenticationFailed -from pulpcore.app import settings - _logger = logging.getLogger(__name__) diff --git a/pulpcore/app/redis_connection.py b/pulpcore/app/redis_connection.py index e612680cbd0..87270c11249 100644 --- a/pulpcore/app/redis_connection.py +++ b/pulpcore/app/redis_connection.py @@ -1,8 +1,7 @@ +from django.conf import settings from redis import Redis from redis.asyncio import Redis as aRedis -from pulpcore.app.settings import settings - _conn = None _a_conn = None diff --git a/pulpcore/app/serializers/exporter.py b/pulpcore/app/serializers/exporter.py index 33339c4eda3..aec8f4adde2 100644 --- a/pulpcore/app/serializers/exporter.py +++ b/pulpcore/app/serializers/exporter.py @@ -2,10 +2,11 @@ import re from gettext import gettext as _ +from django.conf import settings from rest_framework import serializers from rest_framework.validators import UniqueValidator -from pulpcore.app import models, settings +from pulpcore.app import models from pulpcore.app.serializers import ( DetailIdentityField, DetailRelatedField, diff --git a/pulpcore/app/serializers/importer.py b/pulpcore/app/serializers/importer.py index 742ad9ab176..e940f73260a 100644 --- a/pulpcore/app/serializers/importer.py +++ b/pulpcore/app/serializers/importer.py @@ -1,11 +1,12 @@ import os from gettext import gettext as _ +from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from rest_framework import serializers from rest_framework.validators import UniqueValidator -from pulpcore.app import models, settings +from pulpcore.app import models from pulpcore.app.serializers import ( DetailIdentityField, ImportIdentityField, diff --git a/pulpcore/app/serializers/repository.py b/pulpcore/app/serializers/repository.py index f6c75b57c74..7b1267a64d5 100644 --- a/pulpcore/app/serializers/repository.py +++ b/pulpcore/app/serializers/repository.py @@ -3,10 +3,11 @@ from urllib.parse import urlparse from cryptography.x509 import load_pem_x509_certificate +from django.conf import settings from rest_framework import fields, serializers from rest_framework_nested.serializers import NestedHyperlinkedModelSerializer -from pulpcore.app import models, settings +from pulpcore.app import models from pulpcore.app.serializers import ( DetailIdentityField, DetailRelatedField, diff --git a/pulpcore/app/settings.py b/pulpcore/app/settings.py index 1eebae44057..8e1fd0e8328 100644 --- a/pulpcore/app/settings.py +++ b/pulpcore/app/settings.py @@ -11,13 +11,13 @@ import sys from contextlib import suppress from importlib import import_module -from logging import getLogger from pathlib import Path from cryptography.fernet import Fernet from django.conf import global_settings from django.core.exceptions import ImproperlyConfigured from django.core.files.storage import storages +from dynaconf import DjangoDynaconf, Dynaconf, Validator from pulpcore import constants @@ -35,6 +35,24 @@ else: from importlib.metadata import entry_points +# Load settings first pass before applying all the defaults to get a grip on ENABLED_PLUGINS. +enabled_plugins_validator = Validator( + "ENABLED_PLUGINS", + is_type_of=list, + len_min=1, + when=Validator("ENABLED_PLUGINS", must_exist=True), +) + +preload_settings = Dynaconf( + ENVVAR_PREFIX_FOR_DYNACONF="PULP", + ENV_SWITCHER_FOR_DYNACONF="PULP_ENV", + ENVVAR_FOR_DYNACONF="PULP_SETTINGS", + load_dotenv=False, + validators=[ + enabled_plugins_validator, + ], +) + # Build paths inside the project like this: BASE_DIR / ... BASE_DIR = Path(__file__).absolute().parent @@ -89,9 +107,6 @@ # List of upload handler classes to be applied in order. FILE_UPLOAD_HANDLERS = ("pulpcore.app.files.HashingFileUploadHandler",) -# SECURITY WARNING: this should be set to a unique, unpredictable value -SECRET_KEY = "SECRET" - # Key used to encrypt fields in the database DB_ENCRYPTION_KEY = "/etc/pulp/certs/database_fields.symmetric.key" @@ -132,6 +147,15 @@ import_module(app) INSTALLED_APPS.append(app) +# Select enabled plugins and prepare to load their settings. +enabled_plugins = preload_settings.get("ENABLED_PLUGINS", None) +plugin_settings = [] +for entry_point in entry_points(group="pulpcore.plugin"): + if enabled_plugins is None or entry_point.name in enabled_plugins: + plugin_app = entry_point.load() + plugin_settings.append(f"{entry_point.module}.app.settings") + INSTALLED_APPS += [plugin_app] + MIDDLEWARE = [ "django_guid.middleware.guid_middleware", "django.middleware.security.SecurityMiddleware", @@ -386,17 +410,9 @@ # HERE STARTS DYNACONF EXTENSION LOAD (Keep at the very bottom of settings.py) # Read more at https://www.dynaconf.com/django/ -from dynaconf import DjangoDynaconf, Dynaconf, Validator # noqa # Validators -enabled_plugins_validator = Validator( - "ENABLED_PLUGINS", - is_type_of=list, - len_min=1, - when=Validator("ENABLED_PLUGINS", must_exist=True), -) - storage_keys = ("STORAGES.default.BACKEND", "DEFAULT_FILE_STORAGE") storage_validator = ( Validator("REDIRECT_TO_OBJECT_STORAGE", eq=False) @@ -493,39 +509,76 @@ def otel_middleware_hook(settings): return data +def api_root_hook(settings): + # protocol://host:port/{API_ROOT}{domain}/api/{version}/ + # All of the below are DEPRECATED, and should be replaced by calling + # pulpcore.plugin.find_url.find_api_root() (q.v.) + if settings.API_ROOT_REWRITE_HEADER: + api_root = "//" + else: + api_root = settings.API_ROOT + return { + "V3_API_ROOT": api_root + "api/v3/", + "V3_DOMAIN_API_ROOT": api_root + "/api/v3/", + "V3_API_ROOT_NO_FRONT_SLASH": (api_root + "api/v3/").lstrip("/"), + "V3_DOMAIN_API_ROOT_NO_FRONT_SLASH": (api_root + "/api/v3/").lstrip("/"), + } + + +def forbidden_checksums_hook(settings): + return { + "FORBIDDEN_CHECKSUMS": sorted( + set(constants.ALL_KNOWN_CONTENT_CHECKSUMS).difference( + settings.ALLOWED_CONTENT_CHECKSUMS + ) + ), + } + + +def validate_db_encryption_key_hook(settings): + if Path(sys.argv[0]).name in ["pytest", "sphinx-build"] or ( + len(sys.argv) >= 2 and sys.argv[1] in ["collectstatic", "openapi"] + ): + return {} + try: + with open(settings.DB_ENCRYPTION_KEY, "rb") as key_file: + Fernet(key_file.read()) + except Exception as ex: + raise ImproperlyConfigured( + "Could not load DB_ENCRYPTION_KEY file '{file}': {err}".format( + file=settings.DB_ENCRYPTION_KEY, err=ex + ) + ) + return {} + + +del preload_settings + settings = DjangoDynaconf( __name__, ENVVAR_PREFIX_FOR_DYNACONF="PULP", ENV_SWITCHER_FOR_DYNACONF="PULP_ENV", ENVVAR_FOR_DYNACONF="PULP_SETTINGS", + # Ensure the plugin defaults are loaded before user configs. + PRELOAD_FOR_DYNACONF=plugin_settings, load_dotenv=False, validators=[ api_root_validator, cache_validator, - enabled_plugins_validator, sha256_validator, storage_validator, unknown_algs_validator, json_header_auth_validator, authentication_json_header_openapi_security_scheme_validator, ], - post_hooks=(otel_middleware_hook,), + post_hooks=( + otel_middleware_hook, + api_root_hook, + forbidden_checksums_hook, + validate_db_encryption_key_hook, + ), ) -# Select enabled plugins and load their settings. -enabled_plugins = settings.get("ENABLED_PLUGINS", None) -plugin_settings = [] -for entry_point in entry_points(group="pulpcore.plugin"): - if enabled_plugins and entry_point.name not in enabled_plugins: - continue - if (plugin_app := entry_point.load()) not in settings.INSTALLED_APPS: - plugin_settings.append(f"{entry_point.module}.app.settings") - settings.INSTALLED_APPS += [plugin_app] -# Ensure the plugin defaults are loaded before user configs -settings.PRELOAD_FOR_DYNACONF = plugin_settings -settings.reload() -INSTALLED_APPS = settings.INSTALLED_APPS - # begin compatibility layer for DEFAULT_FILE_STORAGE # Remove on pulpcore=3.85 or pulpcore=4.0 @@ -534,33 +587,4 @@ def otel_middleware_hook(settings): storages.backends # end compatibility layer -_logger = getLogger(__name__) - - -if not ( - Path(sys.argv[0]).name in ["pytest", "sphinx-build"] - or (len(sys.argv) >= 2 and sys.argv[1] in ["collectstatic", "openapi"]) -): - try: - with open(DB_ENCRYPTION_KEY, "rb") as key_file: - Fernet(key_file.read()) - except Exception as ex: - raise ImproperlyConfigured( - ("Could not load DB_ENCRYPTION_KEY file '{file}': {err}").format( - file=DB_ENCRYPTION_KEY, err=ex - ) - ) - - -FORBIDDEN_CHECKSUMS = set(constants.ALL_KNOWN_CONTENT_CHECKSUMS).difference( - ALLOWED_CONTENT_CHECKSUMS -) - -if settings.API_ROOT_REWRITE_HEADER: - api_root = "//" -else: - api_root = settings.API_ROOT -settings.set("V3_API_ROOT", api_root + "api/v3/") # Not user configurable -settings.set("V3_DOMAIN_API_ROOT", api_root + "/api/v3/") -settings.set("V3_API_ROOT_NO_FRONT_SLASH", settings.V3_API_ROOT.lstrip("/")) -settings.set("V3_DOMAIN_API_ROOT_NO_FRONT_SLASH", settings.V3_DOMAIN_API_ROOT.lstrip("/")) +# HERE ENDS DYNACONF EXTENSION LOAD (No more code below this line) diff --git a/pulpcore/app/views/importer.py b/pulpcore/app/views/importer.py index 5e5f104588a..e49aa43ba04 100644 --- a/pulpcore/app/views/importer.py +++ b/pulpcore/app/views/importer.py @@ -2,11 +2,11 @@ import os from gettext import gettext as _ +from django.conf import settings from drf_spectacular.utils import extend_schema from rest_framework.response import Response from rest_framework.views import APIView -from pulpcore.app import settings from pulpcore.app.serializers import PulpImportCheckResponseSerializer, PulpImportCheckSerializer diff --git a/pulpcore/cache/cache.py b/pulpcore/cache/cache.py index 7c29fd8b0d5..1fe0c46d806 100644 --- a/pulpcore/cache/cache.py +++ b/pulpcore/cache/cache.py @@ -5,6 +5,7 @@ from aiohttp.web import FileResponse, HTTPSuccessful, Request, Response from aiohttp.web_exceptions import HTTPFound +from django.conf import settings from django.http import FileResponse as ApiFileResponse from django.http import HttpResponse, HttpResponseRedirect from redis import ConnectionError @@ -16,7 +17,6 @@ get_async_redis_connection, get_redis_connection, ) -from pulpcore.app.settings import settings from pulpcore.metrics import artifacts_size_counter from pulpcore.responses import ArtifactResponse diff --git a/pulpcore/tasking/kafka.py b/pulpcore/tasking/kafka.py index 3f6f0fd447d..808e20d7bbf 100644 --- a/pulpcore/tasking/kafka.py +++ b/pulpcore/tasking/kafka.py @@ -6,7 +6,7 @@ from django.conf import settings -_bootstrap_servers = settings.get("KAFKA_BOOTSTRAP_SERVERS") +_bootstrap_servers = getattr(settings, "KAFKA_BOOTSTRAP_SERVERS") if _bootstrap_servers is None: diff --git a/pulpcore/tests/functional/api/test_auth.py b/pulpcore/tests/functional/api/test_auth.py index 3dcd2d7fe9d..e8cd326883c 100644 --- a/pulpcore/tests/functional/api/test_auth.py +++ b/pulpcore/tests/functional/api/test_auth.py @@ -8,8 +8,7 @@ from base64 import b64encode import pytest - -from pulpcore.app import settings +from django.conf import settings @pytest.mark.parallel diff --git a/pulpcore/tests/functional/api/using_plugin/test_filesystemexport.py b/pulpcore/tests/functional/api/using_plugin/test_filesystemexport.py index 4d0fb8ed7e5..fd26d0d14c6 100644 --- a/pulpcore/tests/functional/api/using_plugin/test_filesystemexport.py +++ b/pulpcore/tests/functional/api/using_plugin/test_filesystemexport.py @@ -8,8 +8,8 @@ import uuid import pytest +from django.conf import settings -from pulpcore.app import settings from pulpcore.client.pulp_file import RepositorySyncURL from pulpcore.client.pulpcore.exceptions import ApiException diff --git a/pulpcore/tests/functional/api/using_plugin/test_pulpimport.py b/pulpcore/tests/functional/api/using_plugin/test_pulpimport.py index ef34be5a060..d3067da5069 100644 --- a/pulpcore/tests/functional/api/using_plugin/test_pulpimport.py +++ b/pulpcore/tests/functional/api/using_plugin/test_pulpimport.py @@ -11,8 +11,8 @@ from pathlib import Path import pytest +from django.conf import settings -from pulpcore.app import settings from pulpcore.client.pulp_file import RepositorySyncURL from pulpcore.client.pulpcore.exceptions import ApiException diff --git a/pyproject.toml b/pyproject.toml index 9c4a4431fbd..962219f1900 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -270,7 +270,6 @@ filename = "./pyproject.toml" search = "version = \"{current_version}\"" replace = "version = \"{new_version}\"" - [tool.ruff] # This section is managed by the plugin template. Do not edit manually. line-length = 100 @@ -292,6 +291,7 @@ extend-select = [ [tool.ruff.lint.flake8-tidy-imports.banned-api] # This section is managed by the plugin template. Do not edit manually. "distutils".msg = "The 'distutils' module has been deprecated since Python 3.9." +"pulpcore.app.settings".msg = "Always import 'settings' from 'django.conf' instead." [tool.ruff.lint.isort] # This section is managed by the plugin template. Do not edit manually.