Migrate to Django 5 STORAGES and get_domain()

Assisted by: cursor composer-1.5

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: 2 people

.ci/assets/ci_constraints.txt

@@ -1,5 +1,6 @@
 # Pulpcore versions without the openapi command do no longer work in the CI
-pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*
+# Pulpcore versions without the django 5 storage compatibility will fail, >3.63,<3.70
+pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*,!=3.64.*,!=3.65.*,!=3.66.*,!=3.67.*,!=3.68.*,!=3.69.*
 
 
 tablib!=3.6.0

.ci/scripts/check_release.py

@@ -1,11 +1,20 @@
 #!/usr/bin/env python
+# /// script
+# requires-python = ">=3.13"
+# dependencies = [
+#     "gitpython>=3.1.46,<3.2.0",
+#     "packaging>=26.0,<26.1",
+#     "pyyaml>=6.0.3,<6.1.0",
+# ]
+# ///
 
 import argparse
 import re
 import os
 import tomllib
-import yaml
 from pathlib import Path
+
+import yaml
 from packaging.version import Version
 from git import Repo
 

.github/workflows/ci.yml

@@ -94,13 +94,13 @@ jobs:
       run_docs: ${{ needs.check-changes.outputs.run_docs }}
 
   lint:
-    needs:
-      - "check-changes"
-    if: needs.check-changes.outputs.run_tests == '1'
     uses: "./.github/workflows/lint.yml"
 
   build:
-    needs: "lint"
+    needs:
+      - "check-changes"
+      - "lint"
+    if: needs.check-changes.outputs.run_tests == '1'
     uses: "./.github/workflows/build.yml"
 
   test:

.github/workflows/scripts/install.sh

@@ -95,7 +95,7 @@ if [ "$TEST" = "s3" ]; then
   sed -i -e '$a s3_test: true\
 minio_access_key: "'$MINIO_ACCESS_KEY'"\
 minio_secret_key: "'$MINIO_SECRET_KEY'"\
-pulp_scenario_settings: {"flatpak_index": false, "token_auth_disabled": true}\
+pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "flatpak_index": false, "token_auth_disabled": true}\
 pulp_scenario_env: {}\
 ' vars/main.yaml
   export PULP_API_ROOT="/rerouted/djnd/"
@@ -109,7 +109,7 @@ if [ "$TEST" = "azure" ]; then
       - ./azurite:/etc/pulp\
     command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"' vars/main.yaml
   sed -i -e '$a azure_test: true\
-pulp_scenario_settings: {"flatpak_index": true}\
+pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "flatpak_index": true}\
 pulp_scenario_env: {}\
 ' vars/main.yaml
 fi

pulp_container/app/registry.py

@@ -12,12 +12,12 @@
 from aiohttp.web_exceptions import HTTPTooManyRequests
 from django_guid import set_guid
 from django_guid.utils import generate_guid
-from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import IntegrityError
 from multidict import MultiDict
 
 from pulpcore.plugin.content import Handler, PathNotResolved
+from pulpcore.plugin.util import get_domain
 from pulpcore.plugin.models import RemoteArtifact, Content, ContentArtifact
 from pulpcore.plugin.content import ArtifactResponse
 from pulpcore.plugin.tasking import dispatch
@@ -83,9 +83,10 @@ async def _dispatch(artifact, headers):
         full_headers["Docker-Content-Digest"] = headers["Docker-Content-Digest"]
         full_headers["Docker-Distribution-API-Version"] = "registry/2.0"
 
-        if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem":
+        domain = get_domain()
+        if domain.storage_class == "pulpcore.app.models.storage.FileSystem":
             file = artifact.file
-            path = os.path.join(settings.MEDIA_ROOT, file.name)
+            path = os.path.join(domain.get_storage().location, file.name)
             if not os.path.exists(path):
                 raise Exception("Expected path '{}' is not found".format(path))
             return web.FileResponse(path, headers=full_headers)

pulp_container/app/registry_api.py

@@ -18,7 +18,6 @@
 from tempfile import NamedTemporaryFile
 
 from django.core.exceptions import ObjectDoesNotExist
-from django.core.files.storage import default_storage as storage
 from django.core.files.base import ContentFile, File
 from django.db import IntegrityError, transaction
 from django.db.models import F, Value
@@ -30,7 +29,7 @@
 from pulpcore.plugin.models import Artifact, ContentArtifact, UploadChunk
 from pulpcore.plugin.files import PulpTemporaryUploadedFile
 from pulpcore.plugin.tasking import add_and_remove, dispatch
-from pulpcore.plugin.util import get_objects_for_user, get_url
+from pulpcore.plugin.util import get_objects_for_user, get_url, get_domain
 from pulpcore.plugin.exceptions import TimeoutException
 
 from rest_framework.exceptions import (
@@ -596,7 +595,7 @@ def get_manifest_config(self, manifest):
                 except Artifact.DoesNotExist:
                     log.warning(f"Manifest {manifest.pk}'s config blob was not found.")
                 else:
-                    with storage.open(config_artifact.file.name) as file:
+                    with get_domain().get_storage().open(config_artifact.file.name) as file:
                         raw_data = file.read()
                     config_data = json.loads(raw_data)
                     config["labels"] = config_data.get("config", {}).get("Labels")
@@ -989,14 +988,15 @@ def __init__(self, *args, **kwargs):
         """
         super().__init__(*args, **kwargs)
 
+        domain = get_domain()
         if (
-            settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem"
-            or not settings.REDIRECT_TO_OBJECT_STORAGE
+            domain.storage_class == "pulpcore.app.models.storage.FileSystem"
+            or not domain.redirect_to_object_storage
         ):
             self.redirects_class = FileStorageRedirects
-        elif settings.DEFAULT_FILE_STORAGE == "storages.backends.s3boto3.S3Boto3Storage":
+        elif domain.storage_class == "storages.backends.s3boto3.S3Boto3Storage":
             self.redirects_class = S3StorageRedirects
-        elif settings.DEFAULT_FILE_STORAGE == "storages.backends.azure_storage.AzureStorage":
+        elif domain.storage_class == "storages.backends.azure_storage.AzureStorage":
             self.redirects_class = AzureStorageRedirects
         else:
             raise NotImplementedError()
@@ -1195,6 +1195,8 @@ def put(self, request, path, pk=None):
         """
         Responds with the actual manifest
         """
+        domain = get_domain()
+        storage = domain.get_storage()
         # iterate over all the layers and create
         chunk = request.META["wsgi.input"]
         artifact = self.receive_artifact(chunk)

pulp_container/app/tasks/sign.py

@@ -8,6 +8,7 @@
 from django.db.models import Q
 
 from pulpcore.plugin.models import Repository
+from pulpcore.plugin.util import get_domain
 
 from pulp_container.app.models import (
     ManifestSignature,
@@ -106,7 +107,7 @@ async def create_signature(manifest, reference, signing_service):
         if not manifest.data:
             # TODO: BACKWARD COMPATIBILITY - remove after fully migrating to artifactless manifest
             artifact = await manifest._artifacts.aget()
-            if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem":
+            if get_domain().storage_class != "pulpcore.app.models.storage.FileSystem":
                 async with tempfile.NamedTemporaryFile(dir=".", mode="wb", delete=False) as tf:
                     await tf.write(await sync_to_async(artifact.file.read)())
                     await tf.flush()

pulp_container/app/utils.py

@@ -10,7 +10,7 @@
 
 from asgiref.sync import sync_to_async
 from jsonschema import Draft7Validator, validate, ValidationError
-from django.core.files.storage import default_storage as storage
+from pulpcore.plugin.util import get_domain
 from django.db import IntegrityError
 from functools import partial
 from rest_framework.exceptions import Throttled
@@ -310,6 +310,7 @@ async def save_artifact(artifact_attributes):
 
 
 def get_content_data(saved_artifact):
+    storage = get_domain().get_storage()
     with storage.open(saved_artifact.file.name, mode="rb") as file:
         raw_data = file.read()
     content_data = json.loads(raw_data)

pulp_container/tests/functional/api/test_content_cache.py

@@ -30,13 +30,7 @@
 
 from pulp_container.constants import MEDIA_TYPE
 
-STANDARD_FILE_STORAGE_FRAMEWORKS = [
-    "django.core.files.storage.FileSystemStorage",
-    "pulpcore.app.models.storage.FileSystem",
-]
-
 cli_client = cli.Client(config.get_config())
-DEFAULT_FILE_STORAGE = utils.get_pulp_setting(cli_client, "DEFAULT_FILE_STORAGE")
 CACHE_ENABLED = utils.get_pulp_setting(cli_client, "CACHE_ENABLED")
 
 PULP_CONTENT_HOST_BASE_URL = config.get_config().get_base_url()
@@ -190,13 +184,10 @@ def check_cache(self, url, expected_metadata, headers):
 
     def fetch_response_metadata(self, response):
         """Retrieve metadata from the passed response and normalize status code for redirects."""
-        if DEFAULT_FILE_STORAGE in STANDARD_FILE_STORAGE_FRAMEWORKS:
-            return response.status_code, response.headers.get("X-PULP-CACHE")
-        else:
-            if response.history:
-                response = response.history[0]
-                response.status_code = 200
-            return response.status_code, response.headers.get("X-PULP-CACHE")
+        if response.history:
+            response = response.history[0]
+            response.status_code = 200
+        return response.status_code, response.headers.get("X-PULP-CACHE")
 
 
 def cache_status_first_func(i):

template_config.yml

@@ -2,18 +2,21 @@
 # were not present before running plugin-template have been added with their default values.
 
 # generated with plugin_template
+#
+# After editing this file please always reapply the plugin template before committing any changes.
 
-api_root: /pulp/
+---
+api_root: "/pulp/"
 black: true
 check_commit_message: true
 check_gettext: true
 check_manifest: true
 check_stray_pulpcore_imports: true
-ci_base_image: ghcr.io/pulp/pulp-ci-centos9
+ci_base_image: "ghcr.io/pulp/pulp-ci-centos9"
 ci_env: {}
-ci_trigger: '{pull_request: {branches: [''*'']}}'
-cli_package: pulp-cli
-cli_repo: https://github.com/pulp/pulp-cli.git
+ci_trigger: "{pull_request: {branches: ['*']}}"
+cli_package: "pulp-cli"
+cli_repo: "https://github.com/pulp/pulp-cli.git"
 core_import_allowed: []
 deploy_client_to_pypi: true
 deploy_client_to_rubygems: true
@@ -23,54 +26,81 @@ docker_fixtures: false
 extra_files: []
 flake8: true
 flake8_ignore: []
-github_org: pulp
-latest_release_branch: '2.21'
+github_org: "pulp"
+latest_release_branch: "2.21"
 lint_requirements: true
 os_required_packages: []
 parallel_test_workers: 8
-plugin_app_label: container
-plugin_default_branch: main
-plugin_name: pulp_container
+plugin_app_label: "container"
+plugin_default_branch: "main"
+plugin_name: "pulp_container"
 plugins:
-- app_label: container
-  name: pulp_container
-post_job_template: null
-pre_job_template: null
+  - app_label: "container"
+    name: "pulp_container"
 pulp_env: {}
 pulp_env_azure: {}
 pulp_env_gcp: {}
 pulp_env_s3: {}
-pulp_scheme: https
+pulp_scheme: "https"
 pulp_settings:
   allowed_content_checksums:
-  - sha1
-  - sha224
-  - sha256
-  - sha384
-  - sha512
+    - "sha1"
+    - "sha224"
+    - "sha256"
+    - "sha384"
+    - "sha512"
   allowed_export_paths:
-  - /tmp
+    - "/tmp"
   allowed_import_paths:
-  - /tmp
+    - "/tmp"
   flatpak_index: true
 pulp_settings_azure:
+  MEDIA_ROOT: ""
+  STORAGES:
+    default:
+      BACKEND: "storages.backends.azure_storage.AzureStorage"
+      OPTIONS:
+        account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+        account_name: "devstoreaccount1"
+        azure_container: "pulp-test"
+        connection_string: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;"
+        expiration_secs: 120
+        location: "pulp3"
+        overwrite_files: true
+    staticfiles:
+      BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage"
   flatpak_index: true
 pulp_settings_gcp: null
 pulp_settings_s3:
+  MEDIA_ROOT: ""
+  STORAGES:
+    default:
+      BACKEND: "storages.backends.s3boto3.S3Boto3Storage"
+      OPTIONS:
+        access_key: "AKIAIT2Z5TDYPX3ARJBA"
+        addressing_style: "path"
+        bucket_name: "pulp3"
+        default_acl: "@none"
+        endpoint_url: "http://minio:9000"
+        region_name: "eu-central-1"
+        secret_key: "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS"
+        signature_version: "s3v4"
+    staticfiles:
+      BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage"
   flatpak_index: false
   token_auth_disabled: true
 pydocstyle: true
-release_email: pulp-infra@redhat.com
-release_user: pulpbot
+release_email: "pulp-infra@redhat.com"
+release_user: "pulpbot"
 stalebot: true
 stalebot_days_until_close: 30
 stalebot_days_until_stale: 90
 stalebot_limit_to_pulls: true
 supported_release_branches:
-- '2.14'
-- '2.15'
-- '2.16'
-- '2.20'
+  - "2.14"
+  - "2.15"
+  - "2.16"
+  - "2.20"
 sync_ci: true
 test_azure: true
 test_cli: true
@@ -81,4 +111,5 @@ test_performance: false
 test_reroute: true
 test_s3: true
 use_issue_template: true
+...