feat(cache): L1 singleton settings cache + Celery-scoped authorization caching

docker-compose.override.unit_tests.yml

@@ -22,6 +22,11 @@ services:
       DD_DATABASE_USER: ${DD_DATABASE_USER:-defectdojo}
       DD_DATABASE_PASSWORD: ${DD_DATABASE_PASSWORD:-defectdojo}
       DD_CELERY_BROKER_URL: 'sqla+sqlite:///dojo.celerydb.sqlite'
+      # No Redis/valkey in unit tests -> default django cache is LocMemCache.
+      DD_CACHE_URL: ''
+      # In-process singleton cache (dojo/caching.py) stays ON for deterministic
+      # assertNumQueries counts; reset per request (middleware) and per test.
+      DD_SETTINGS_CACHE_L1_TTL: '30'
       DD_JIRA_EXTRA_ISSUE_TYPES: 'Vulnerability' # Shouldn't trigger a migration error
   celerybeat: !reset
   celeryworker: !reset

docker-compose.override.unit_tests_cicd.yml

@@ -23,6 +23,12 @@ services:
       DD_CELERY_BROKER_URL: 'sqla+sqlite:///dojo.celerydb.sqlite'
       DD_JIRA_EXTRA_ISSUE_TYPES: 'Vulnerability' # Shouldn't trigger a migration error
       DD_V3_FEATURE_LOCATIONS: ${DD_V3_FEATURE_LOCATIONS:-False}
+      # No Redis/valkey in unit tests -> default django cache is LocMemCache.
+      DD_CACHE_URL: ''
+      # In-process singleton cache (dojo/caching.py) stays ON: a singleton is read
+      # once per request/test (deterministic assertNumQueries), reset per request
+      # (middleware) and per test (dojo_test_case setUp).
+      DD_SETTINGS_CACHE_L1_TTL: '30'
   celerybeat: !reset
   celeryworker: !reset
   initializer: !reset

docker-compose.yml

@@ -50,6 +50,7 @@ services:
       DD_ALLOWED_HOSTS: "${DD_ALLOWED_HOSTS:-*}"
       DD_DATABASE_URL: ${DD_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/defectdojo}
       DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL:-redis://valkey:6379/0}
+      DD_CACHE_URL: ${DD_CACHE_URL:-redis://valkey:6379/1}
       DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}"
       DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}"
       DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}"
@@ -71,6 +72,7 @@ services:
     environment:
       DD_DATABASE_URL: ${DD_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/defectdojo}
       DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL:-redis://valkey:6379/0}
+      DD_CACHE_URL: ${DD_CACHE_URL:-redis://valkey:6379/1}
       DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}"
       DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}"
       DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}"
@@ -91,6 +93,7 @@ services:
     environment:
       DD_DATABASE_URL: ${DD_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/defectdojo}
       DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL:-redis://valkey:6379/0}
+      DD_CACHE_URL: ${DD_CACHE_URL:-redis://valkey:6379/1}
       DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}"
       DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}"
       DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}"

dojo/authorization/query_registrations.py

@@ -40,7 +40,7 @@
     Tool_Product_Settings,
     Vulnerability_Id,
 )
-from dojo.request_cache import cache_for_request
+from dojo.request_cache import cache_for_request_or_task
 
 
 def _resolve_user(user):
@@ -81,7 +81,7 @@ def _authorized_product_type_ids(user):
     return Product_Type.objects.filter(authorized_users=user).values("id")
 
 
-@cache_for_request
+@cache_for_request_or_task
 def authorized_product_id_set(user_pk):
     """
     Frozen set of product ids the user can access via authorized_users
@@ -101,7 +101,7 @@ def authorized_product_id_set(user_pk):
     )
 
 
-@cache_for_request
+@cache_for_request_or_task
 def authorized_product_type_id_set(user_pk):
     """
     Frozen set of product_type ids the user is a direct member of via

dojo/authorization/template_filters.py

@@ -2,7 +2,7 @@
 
 from dojo.authorization.authorization import user_has_configuration_permission as configuration_permission
 from dojo.authorization.authorization import user_has_global_permission, user_has_permission
-from dojo.request_cache import cache_for_request
+from dojo.request_cache import cache_for_request_or_task
 
 
 def has_object_permission(obj, permission):
@@ -21,7 +21,7 @@ def has_configuration_permission(permission, request):
     return configuration_permission(user, permission)
 
 
-@cache_for_request
+@cache_for_request_or_task
 def get_user_permissions(user):
     return user.user_permissions.all()
 
@@ -31,7 +31,7 @@ def user_has_configuration_permission_without_group(user, codename):
     return any(permission.codename == codename for permission in permissions)
 
 
-@cache_for_request
+@cache_for_request_or_task
 def get_group_permissions(group):
     return group.permissions.all()
 

dojo/caching.py

@@ -0,0 +1,202 @@
+"""
+In-process read-through cache for global, low-cardinality singleton config.
+
+A single per-thread **L1** tier resolves a getter L1 → DB: a hit wins; a ``None``
+result means "not cached, compute it" and is never stored (unless the getter opts
+in with ``cache_none=True``, for getters where ``None`` is a legitimate answer).
+This is deliberately simple — it is only for global, user-INDEPENDENT,
+signal-invalidated singletons (feature flags, system settings, and the like),
+never per-user or per-object data.
+
+There is intentionally **no shared/cross-process (L2) tier**: freshness is provided
+by resetting L1 at every request and task boundary (middleware + the Celery task
+base), so each request/task reads the singleton from the DB at most once and never
+serves a value cached during a prior request/task (e.g. a since-changed
+``System_Settings``). This keeps the design free of a Redis dependency, pickled
+model graphs, and cross-process invalidation — at the cost of one DB read per
+singleton per request/task. (The default ``django.core.cache`` backend may still be
+Redis for other uses; this module no longer reads or writes it.)
+
+Values are stored as plain dicts/scalars (see ``model_to_cache_dict`` /
+``cache_dict_to_model``), never pickled model instances.
+
+Configuration (Django setting, wired from env in ``settings.dist.py``):
+
+* ``SETTINGS_CACHE_L1_TTL`` — per-thread in-process freshness budget in seconds
+  (``-1`` disables the cache, making the decorator a pass-through). Keep it short.
+  L1 is reset at each request/task boundary, so it is effectively request/task
+  scoped.
+"""
+
+import threading
+import time
+from functools import wraps
+
+from django.conf import settings
+
+
+class _L1Store:
+
+    """
+    Per-thread in-process store, TTL-stamped. ``get`` returns the value or ``None``
+    (absent, expired, or L1 disabled via ``SETTINGS_CACHE_L1_TTL`` < 0).
+
+    Per-thread (not shared across threads) so it needs no locking, and is reset at
+    each request/task boundary (see ``reset``) — making it effectively request/task
+    scoped on a reused worker/uwsgi thread.
+    """
+
+    def __init__(self):
+        self._local = threading.local()
+
+    def _bucket(self):
+        bucket = getattr(self._local, "b", None)
+        if bucket is None:
+            bucket = self._local.b = {}
+        return bucket
+
+    def get(self, key):
+        if getattr(settings, "SETTINGS_CACHE_L1_TTL", 30) < 0:
+            return None
+        entry = self._bucket().get(key)
+        if entry is None:
+            return None
+        value, expiry = entry
+        if time.monotonic() >= expiry:
+            self._bucket().pop(key, None)
+            return None
+        return value
+
+    def set(self, key, value):
+        ttl = getattr(settings, "SETTINGS_CACHE_L1_TTL", 30)
+        if ttl < 0:
+            return
+        self._bucket()[key] = (value, time.monotonic() + ttl)
+
+    def invalidate(self, key):
+        # Only this thread; other threads/processes self-heal within the L1 TTL
+        # (and reset at their next request/task boundary).
+        self._bucket().pop(key, None)
+
+    def reset(self):
+        # Clear THIS thread's L1 bucket. Called at request/task boundaries so a
+        # reused worker/uwsgi thread never serves a value cached during a prior
+        # request or task (e.g. a since-changed System_Settings).
+        self._bucket().clear()
+
+    def clear(self):
+        # Test helper: drop this thread's entries.
+        self._local = threading.local()
+
+
+_L1_STORE = _L1Store()
+
+# Sentinel stored in L1 to represent a cached ``None`` result, so a legitimately
+# ``None`` value (e.g. "no default configured") is distinguishable from "absent".
+# Only used by getters that opt in via ``cache_none=True``.
+_CACHED_NONE = object()
+
+
+def dojo_settings_cache(*, key: str, cache_none: bool = False):
+    """
+    Read-through in-process (L1) cache for a fixed-key singleton getter.
+
+    Resolves L1 → wrapped function. Becomes a pass-through when L1 is disabled
+    (``SETTINGS_CACHE_L1_TTL=-1``). Freshness across processes comes from resetting
+    L1 each request/task (see ``reset_l1_cache``), not a shared tier.
+
+    By default a ``None`` result is treated as "no value" and is not cached (so the
+    next call retries) — right for singletons that always exist and where ``None``
+    means "not yet computed / transient error".
+
+    Pass ``cache_none=True`` for getters where ``None`` is a legitimate steady-state
+    answer (e.g. "no default row configured"). It caches the ``None`` too, so a
+    missing row costs one DB read per request/task instead of one per call site.
+    Safe only when the getter is signal-invalidated on the row's create/save (so the
+    cached ``None`` is dropped the moment a value appears).
+    """
+
+    def decorator(fn):
+        @wraps(fn)
+        def wrapper(*args, **kwargs):
+            value = _L1_STORE.get(key)              # ---- L1 ----
+            if value is _CACHED_NONE:               # cached "no value" (cache_none=True)
+                return None
+            if value is not None:
+                return value
+
+            value = fn(*args, **kwargs)             # ---- miss: compute ----
+            if value is not None:
+                _L1_STORE.set(key, value)
+            elif cache_none:
+                _L1_STORE.set(key, _CACHED_NONE)
+            return value
+
+        return wrapper
+
+    return decorator
+
+
+def invalidate_dojo_settings_cache(key: str) -> None:
+    """
+    Drop a cached singleton from L1 (this thread).
+
+    With no shared tier, other threads/processes self-heal at their next
+    request/task boundary (L1 reset), so there is nothing cross-process to drop.
+    """
+    _L1_STORE.invalidate(key)
+
+
+def reset_l1_cache() -> None:
+    """
+    Reset the current thread's L1 tier.
+
+    Call at request/task boundaries (reused worker/uwsgi threads) so the
+    in-process L1 is effectively request/task-scoped and never serves a value
+    cached during a prior request or task.
+    """
+    _L1_STORE.reset()
+
+
+def model_to_cache_dict(instance) -> dict:
+    """
+    Flatten a model instance to a plain dict of concrete field values.
+
+    Keyed by ``attname`` (so a relation ``foo`` becomes ``foo_id``) and includes
+    the primary key. M2M and reverse relations are skipped. Storing this instead
+    of the model instance keeps the cache free of pickled model graphs; rebuild a
+    live instance with ``cache_dict_to_model``.
+    """
+    return {f.attname: f.value_from_object(instance) for f in instance._meta.concrete_fields}
+
+
+# Attribute set on instances rebuilt for read-only cache use (``read_only=True``).
+# A guarded model's ``save()`` checks it and refuses to persist a cache-derived
+# snapshot (see ``System_Settings.save``). Instance-level, so it never leaks to a
+# freshly-fetched (``no_cache=True``) instance, which is a distinct object.
+READ_ONLY_CACHE_MARKER = "_dd_read_only_cache_snapshot"
+
+
+class ReadOnlyCachedInstanceError(RuntimeError):
+
+    """
+    Raised when code tries to save a model instance rebuilt from the read-through
+    cache. Such an instance is a point-in-time snapshot; persisting it can clobber
+    concurrent changes with stale field values. Fetch a fresh DB instance
+    (e.g. ``System_Settings.objects.get(no_cache=True)``) before saving.
+    """
+
+
+def cache_dict_to_model(model_cls, data: dict, *, read_only: bool = False):
+    """
+    Rebuild an in-memory model instance from a ``model_to_cache_dict`` dict.
+
+    For read-only use; callers that persist changes must fetch a fresh DB instance
+    rather than saving a cache-derived one. Pass ``read_only=True`` to tag the
+    instance with ``READ_ONLY_CACHE_MARKER`` so a guarded model's ``save()`` fails
+    loudly instead of silently writing a stale snapshot.
+    """
+    instance = model_cls(**data)
+    if read_only:
+        setattr(instance, READ_ONLY_CACHE_MARKER, True)
+    return instance

dojo/celery.py

@@ -28,6 +28,12 @@ def __call__(self, *args, **kwargs):
         """
         Restore user context in the celery worker via crum.impersonate.
 
+        Also resets the request/task-scoped L1 settings cache at the start of every
+        task (including eager): a prefork worker reuses its thread across tasks, so an
+        L1 value cached during a prior task (e.g. System_Settings) would otherwise be
+        served stale even after another process changed and saved it. There is no
+        shared tier, so a reset task re-reads each singleton once from the DB.
+
         The apply_async method injects ``async_user_id`` into kwargs when a task
         is dispatched. Here we pop it, resolve to a user instance, and set it
         as the current user in thread-local storage so that all downstream
@@ -39,17 +45,27 @@ def __call__(self, *args, **kwargs):
         intact so that callers who already set a user (e.g. via
         crum.impersonate in tests or request middleware) are not disrupted.
         """
-        if "async_user_id" not in kwargs:
-            return super().__call__(*args, **kwargs)
-
-        import crum  # noqa: PLC0415
-
-        from dojo.models import Dojo_User  # noqa: PLC0415 circular import
-
-        user_id = kwargs.pop("async_user_id")
-        user = Dojo_User.objects.filter(pk=user_id).first() if user_id else None
-        with crum.impersonate(user):
-            return super().__call__(*args, **kwargs)
+        from dojo.caching import reset_l1_cache  # noqa: PLC0415
+        from dojo.request_cache import begin_task_cache, end_task_cache  # noqa: PLC0415
+        reset_l1_cache()
+        # Install a fresh task-scoped request-cache (begin) and drop it afterwards
+        # (end), so cache_for_request_or_task can memoize within this task without a
+        # value leaking to the next task on this reused worker thread.
+        begin_task_cache()
+        try:
+            if "async_user_id" not in kwargs:
+                return super().__call__(*args, **kwargs)
+
+            import crum  # noqa: PLC0415
+
+            from dojo.models import Dojo_User  # noqa: PLC0415 circular import
+
+            user_id = kwargs.pop("async_user_id")
+            user = Dojo_User.objects.filter(pk=user_id).first() if user_id else None
+            with crum.impersonate(user):
+                return super().__call__(*args, **kwargs)
+        finally:
+            end_task_cache()
 
     def apply_async(self, args=None, kwargs=None, **options):
         """Override apply_async to inject user context and track tasks."""

dojo/endpoint/queries.py

@@ -7,11 +7,11 @@ def get_auth_filter(key): return None
     Endpoint,
     Endpoint_Status,
 )
-from dojo.request_cache import cache_for_request
+from dojo.request_cache import cache_for_request_or_task
 
 
 # Cached: all parameters are hashable, no dynamic queryset filtering
-@cache_for_request
+@cache_for_request_or_task
 def get_authorized_endpoints(permission, user=None):
     impl = get_auth_filter("endpoint.get_authorized_endpoints")
     if impl:
@@ -27,7 +27,7 @@ def get_authorized_endpoints_for_queryset(permission, queryset, user=None):
 
 
 # Cached: all parameters are hashable, no dynamic queryset filtering
-@cache_for_request
+@cache_for_request_or_task
 def get_authorized_endpoint_status(permission, user=None):
     impl = get_auth_filter("endpoint.get_authorized_endpoint_status")
     if impl:

dojo/engagement/queries.py

@@ -4,11 +4,11 @@
     def get_auth_filter(key): return None
 
 from dojo.models import Engagement
-from dojo.request_cache import cache_for_request
+from dojo.request_cache import cache_for_request_or_task
 
 
 # Cached: all parameters are hashable, no dynamic queryset filtering
-@cache_for_request
+@cache_for_request_or_task
 def get_authorized_engagements(permission):
     impl = get_auth_filter("engagement.get_authorized_engagements")
     if impl: