diff --git a/docker-compose.override.unit_tests.yml b/docker-compose.override.unit_tests.yml index 565ff78d955..ba243c56d60 100644 --- a/docker-compose.override.unit_tests.yml +++ b/docker-compose.override.unit_tests.yml @@ -22,6 +22,11 @@ services: DD_DATABASE_USER: ${DD_DATABASE_USER:-defectdojo} DD_DATABASE_PASSWORD: ${DD_DATABASE_PASSWORD:-defectdojo} DD_CELERY_BROKER_URL: 'sqla+sqlite:///dojo.celerydb.sqlite' + # No Redis/valkey in unit tests -> default django cache is LocMemCache. + DD_CACHE_URL: '' + # In-process singleton cache (dojo/caching.py) stays ON for deterministic + # assertNumQueries counts; reset per request (middleware) and per test. + DD_SETTINGS_CACHE_L1_TTL: '30' DD_JIRA_EXTRA_ISSUE_TYPES: 'Vulnerability' # Shouldn't trigger a migration error celerybeat: !reset celeryworker: !reset diff --git a/docker-compose.override.unit_tests_cicd.yml b/docker-compose.override.unit_tests_cicd.yml index 01be14baa27..511f76ebcdb 100644 --- a/docker-compose.override.unit_tests_cicd.yml +++ b/docker-compose.override.unit_tests_cicd.yml @@ -23,6 +23,12 @@ services: DD_CELERY_BROKER_URL: 'sqla+sqlite:///dojo.celerydb.sqlite' DD_JIRA_EXTRA_ISSUE_TYPES: 'Vulnerability' # Shouldn't trigger a migration error DD_V3_FEATURE_LOCATIONS: ${DD_V3_FEATURE_LOCATIONS:-False} + # No Redis/valkey in unit tests -> default django cache is LocMemCache. + DD_CACHE_URL: '' + # In-process singleton cache (dojo/caching.py) stays ON: a singleton is read + # once per request/test (deterministic assertNumQueries), reset per request + # (middleware) and per test (dojo_test_case setUp). + DD_SETTINGS_CACHE_L1_TTL: '30' celerybeat: !reset celeryworker: !reset initializer: !reset diff --git a/docker-compose.yml b/docker-compose.yml index 04358dd9a63..0f815b6b8d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,6 +50,7 @@ services: DD_ALLOWED_HOSTS: "${DD_ALLOWED_HOSTS:-*}" DD_DATABASE_URL: ${DD_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/defectdojo} DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL:-redis://valkey:6379/0} + DD_CACHE_URL: ${DD_CACHE_URL:-redis://valkey:6379/1} DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}" DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}" DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}" @@ -71,6 +72,7 @@ services: environment: DD_DATABASE_URL: ${DD_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/defectdojo} DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL:-redis://valkey:6379/0} + DD_CACHE_URL: ${DD_CACHE_URL:-redis://valkey:6379/1} DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}" DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}" DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}" @@ -91,6 +93,7 @@ services: environment: DD_DATABASE_URL: ${DD_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/defectdojo} DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL:-redis://valkey:6379/0} + DD_CACHE_URL: ${DD_CACHE_URL:-redis://valkey:6379/1} DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}" DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}" DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}" diff --git a/dojo/authorization/query_registrations.py b/dojo/authorization/query_registrations.py index f27aba3950b..0e5f26a5c05 100644 --- a/dojo/authorization/query_registrations.py +++ b/dojo/authorization/query_registrations.py @@ -40,7 +40,7 @@ Tool_Product_Settings, Vulnerability_Id, ) -from dojo.request_cache import cache_for_request +from dojo.request_cache import cache_for_request_or_task def _resolve_user(user): @@ -81,7 +81,7 @@ def _authorized_product_type_ids(user): return Product_Type.objects.filter(authorized_users=user).values("id") -@cache_for_request +@cache_for_request_or_task def authorized_product_id_set(user_pk): """ Frozen set of product ids the user can access via authorized_users @@ -101,7 +101,7 @@ def authorized_product_id_set(user_pk): ) -@cache_for_request +@cache_for_request_or_task def authorized_product_type_id_set(user_pk): """ Frozen set of product_type ids the user is a direct member of via diff --git a/dojo/authorization/template_filters.py b/dojo/authorization/template_filters.py index 4e7caf61ba2..751b50f6f6b 100644 --- a/dojo/authorization/template_filters.py +++ b/dojo/authorization/template_filters.py @@ -2,7 +2,7 @@ from dojo.authorization.authorization import user_has_configuration_permission as configuration_permission from dojo.authorization.authorization import user_has_global_permission, user_has_permission -from dojo.request_cache import cache_for_request +from dojo.request_cache import cache_for_request_or_task def has_object_permission(obj, permission): @@ -21,7 +21,7 @@ def has_configuration_permission(permission, request): return configuration_permission(user, permission) -@cache_for_request +@cache_for_request_or_task def get_user_permissions(user): return user.user_permissions.all() @@ -31,7 +31,7 @@ def user_has_configuration_permission_without_group(user, codename): return any(permission.codename == codename for permission in permissions) -@cache_for_request +@cache_for_request_or_task def get_group_permissions(group): return group.permissions.all() diff --git a/dojo/caching.py b/dojo/caching.py new file mode 100644 index 00000000000..c29fd0bf7d5 --- /dev/null +++ b/dojo/caching.py @@ -0,0 +1,202 @@ +""" +In-process read-through cache for global, low-cardinality singleton config. + +A single per-thread **L1** tier resolves a getter L1 → DB: a hit wins; a ``None`` +result means "not cached, compute it" and is never stored (unless the getter opts +in with ``cache_none=True``, for getters where ``None`` is a legitimate answer). +This is deliberately simple — it is only for global, user-INDEPENDENT, +signal-invalidated singletons (feature flags, system settings, and the like), +never per-user or per-object data. + +There is intentionally **no shared/cross-process (L2) tier**: freshness is provided +by resetting L1 at every request and task boundary (middleware + the Celery task +base), so each request/task reads the singleton from the DB at most once and never +serves a value cached during a prior request/task (e.g. a since-changed +``System_Settings``). This keeps the design free of a Redis dependency, pickled +model graphs, and cross-process invalidation — at the cost of one DB read per +singleton per request/task. (The default ``django.core.cache`` backend may still be +Redis for other uses; this module no longer reads or writes it.) + +Values are stored as plain dicts/scalars (see ``model_to_cache_dict`` / +``cache_dict_to_model``), never pickled model instances. + +Configuration (Django setting, wired from env in ``settings.dist.py``): + +* ``SETTINGS_CACHE_L1_TTL`` — per-thread in-process freshness budget in seconds + (``-1`` disables the cache, making the decorator a pass-through). Keep it short. + L1 is reset at each request/task boundary, so it is effectively request/task + scoped. +""" + +import threading +import time +from functools import wraps + +from django.conf import settings + + +class _L1Store: + + """ + Per-thread in-process store, TTL-stamped. ``get`` returns the value or ``None`` + (absent, expired, or L1 disabled via ``SETTINGS_CACHE_L1_TTL`` < 0). + + Per-thread (not shared across threads) so it needs no locking, and is reset at + each request/task boundary (see ``reset``) — making it effectively request/task + scoped on a reused worker/uwsgi thread. + """ + + def __init__(self): + self._local = threading.local() + + def _bucket(self): + bucket = getattr(self._local, "b", None) + if bucket is None: + bucket = self._local.b = {} + return bucket + + def get(self, key): + if getattr(settings, "SETTINGS_CACHE_L1_TTL", 30) < 0: + return None + entry = self._bucket().get(key) + if entry is None: + return None + value, expiry = entry + if time.monotonic() >= expiry: + self._bucket().pop(key, None) + return None + return value + + def set(self, key, value): + ttl = getattr(settings, "SETTINGS_CACHE_L1_TTL", 30) + if ttl < 0: + return + self._bucket()[key] = (value, time.monotonic() + ttl) + + def invalidate(self, key): + # Only this thread; other threads/processes self-heal within the L1 TTL + # (and reset at their next request/task boundary). + self._bucket().pop(key, None) + + def reset(self): + # Clear THIS thread's L1 bucket. Called at request/task boundaries so a + # reused worker/uwsgi thread never serves a value cached during a prior + # request or task (e.g. a since-changed System_Settings). + self._bucket().clear() + + def clear(self): + # Test helper: drop this thread's entries. + self._local = threading.local() + + +_L1_STORE = _L1Store() + +# Sentinel stored in L1 to represent a cached ``None`` result, so a legitimately +# ``None`` value (e.g. "no default configured") is distinguishable from "absent". +# Only used by getters that opt in via ``cache_none=True``. +_CACHED_NONE = object() + + +def dojo_settings_cache(*, key: str, cache_none: bool = False): + """ + Read-through in-process (L1) cache for a fixed-key singleton getter. + + Resolves L1 → wrapped function. Becomes a pass-through when L1 is disabled + (``SETTINGS_CACHE_L1_TTL=-1``). Freshness across processes comes from resetting + L1 each request/task (see ``reset_l1_cache``), not a shared tier. + + By default a ``None`` result is treated as "no value" and is not cached (so the + next call retries) — right for singletons that always exist and where ``None`` + means "not yet computed / transient error". + + Pass ``cache_none=True`` for getters where ``None`` is a legitimate steady-state + answer (e.g. "no default row configured"). It caches the ``None`` too, so a + missing row costs one DB read per request/task instead of one per call site. + Safe only when the getter is signal-invalidated on the row's create/save (so the + cached ``None`` is dropped the moment a value appears). + """ + + def decorator(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + value = _L1_STORE.get(key) # ---- L1 ---- + if value is _CACHED_NONE: # cached "no value" (cache_none=True) + return None + if value is not None: + return value + + value = fn(*args, **kwargs) # ---- miss: compute ---- + if value is not None: + _L1_STORE.set(key, value) + elif cache_none: + _L1_STORE.set(key, _CACHED_NONE) + return value + + return wrapper + + return decorator + + +def invalidate_dojo_settings_cache(key: str) -> None: + """ + Drop a cached singleton from L1 (this thread). + + With no shared tier, other threads/processes self-heal at their next + request/task boundary (L1 reset), so there is nothing cross-process to drop. + """ + _L1_STORE.invalidate(key) + + +def reset_l1_cache() -> None: + """ + Reset the current thread's L1 tier. + + Call at request/task boundaries (reused worker/uwsgi threads) so the + in-process L1 is effectively request/task-scoped and never serves a value + cached during a prior request or task. + """ + _L1_STORE.reset() + + +def model_to_cache_dict(instance) -> dict: + """ + Flatten a model instance to a plain dict of concrete field values. + + Keyed by ``attname`` (so a relation ``foo`` becomes ``foo_id``) and includes + the primary key. M2M and reverse relations are skipped. Storing this instead + of the model instance keeps the cache free of pickled model graphs; rebuild a + live instance with ``cache_dict_to_model``. + """ + return {f.attname: f.value_from_object(instance) for f in instance._meta.concrete_fields} + + +# Attribute set on instances rebuilt for read-only cache use (``read_only=True``). +# A guarded model's ``save()`` checks it and refuses to persist a cache-derived +# snapshot (see ``System_Settings.save``). Instance-level, so it never leaks to a +# freshly-fetched (``no_cache=True``) instance, which is a distinct object. +READ_ONLY_CACHE_MARKER = "_dd_read_only_cache_snapshot" + + +class ReadOnlyCachedInstanceError(RuntimeError): + + """ + Raised when code tries to save a model instance rebuilt from the read-through + cache. Such an instance is a point-in-time snapshot; persisting it can clobber + concurrent changes with stale field values. Fetch a fresh DB instance + (e.g. ``System_Settings.objects.get(no_cache=True)``) before saving. + """ + + +def cache_dict_to_model(model_cls, data: dict, *, read_only: bool = False): + """ + Rebuild an in-memory model instance from a ``model_to_cache_dict`` dict. + + For read-only use; callers that persist changes must fetch a fresh DB instance + rather than saving a cache-derived one. Pass ``read_only=True`` to tag the + instance with ``READ_ONLY_CACHE_MARKER`` so a guarded model's ``save()`` fails + loudly instead of silently writing a stale snapshot. + """ + instance = model_cls(**data) + if read_only: + setattr(instance, READ_ONLY_CACHE_MARKER, True) + return instance diff --git a/dojo/celery.py b/dojo/celery.py index 81dd44095d3..9f798a8a82b 100644 --- a/dojo/celery.py +++ b/dojo/celery.py @@ -28,6 +28,12 @@ def __call__(self, *args, **kwargs): """ Restore user context in the celery worker via crum.impersonate. + Also resets the request/task-scoped L1 settings cache at the start of every + task (including eager): a prefork worker reuses its thread across tasks, so an + L1 value cached during a prior task (e.g. System_Settings) would otherwise be + served stale even after another process changed and saved it. There is no + shared tier, so a reset task re-reads each singleton once from the DB. + The apply_async method injects ``async_user_id`` into kwargs when a task is dispatched. Here we pop it, resolve to a user instance, and set it as the current user in thread-local storage so that all downstream @@ -39,17 +45,27 @@ def __call__(self, *args, **kwargs): intact so that callers who already set a user (e.g. via crum.impersonate in tests or request middleware) are not disrupted. """ - if "async_user_id" not in kwargs: - return super().__call__(*args, **kwargs) - - import crum # noqa: PLC0415 - - from dojo.models import Dojo_User # noqa: PLC0415 circular import - - user_id = kwargs.pop("async_user_id") - user = Dojo_User.objects.filter(pk=user_id).first() if user_id else None - with crum.impersonate(user): - return super().__call__(*args, **kwargs) + from dojo.caching import reset_l1_cache # noqa: PLC0415 + from dojo.request_cache import begin_task_cache, end_task_cache # noqa: PLC0415 + reset_l1_cache() + # Install a fresh task-scoped request-cache (begin) and drop it afterwards + # (end), so cache_for_request_or_task can memoize within this task without a + # value leaking to the next task on this reused worker thread. + begin_task_cache() + try: + if "async_user_id" not in kwargs: + return super().__call__(*args, **kwargs) + + import crum # noqa: PLC0415 + + from dojo.models import Dojo_User # noqa: PLC0415 circular import + + user_id = kwargs.pop("async_user_id") + user = Dojo_User.objects.filter(pk=user_id).first() if user_id else None + with crum.impersonate(user): + return super().__call__(*args, **kwargs) + finally: + end_task_cache() def apply_async(self, args=None, kwargs=None, **options): """Override apply_async to inject user context and track tasks.""" diff --git a/dojo/endpoint/queries.py b/dojo/endpoint/queries.py index e2b43be1051..0cb0444ca48 100644 --- a/dojo/endpoint/queries.py +++ b/dojo/endpoint/queries.py @@ -7,11 +7,11 @@ def get_auth_filter(key): return None Endpoint, Endpoint_Status, ) -from dojo.request_cache import cache_for_request +from dojo.request_cache import cache_for_request_or_task # Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request +@cache_for_request_or_task def get_authorized_endpoints(permission, user=None): impl = get_auth_filter("endpoint.get_authorized_endpoints") if impl: @@ -27,7 +27,7 @@ def get_authorized_endpoints_for_queryset(permission, queryset, user=None): # Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request +@cache_for_request_or_task def get_authorized_endpoint_status(permission, user=None): impl = get_auth_filter("endpoint.get_authorized_endpoint_status") if impl: diff --git a/dojo/engagement/queries.py b/dojo/engagement/queries.py index b46bd51aebf..1b1e99ff38f 100644 --- a/dojo/engagement/queries.py +++ b/dojo/engagement/queries.py @@ -4,11 +4,11 @@ def get_auth_filter(key): return None from dojo.models import Engagement -from dojo.request_cache import cache_for_request +from dojo.request_cache import cache_for_request_or_task # Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request +@cache_for_request_or_task def get_authorized_engagements(permission): impl = get_auth_filter("engagement.get_authorized_engagements") if impl: diff --git a/dojo/finding/queries.py b/dojo/finding/queries.py index 06711882c42..82adda8773d 100644 --- a/dojo/finding/queries.py +++ b/dojo/finding/queries.py @@ -19,13 +19,13 @@ def get_auth_filter(key): return None Test_Import_Finding_Action, Vulnerability_Id, ) -from dojo.request_cache import cache_for_request +from dojo.request_cache import cache_for_request_or_task logger = logging.getLogger(__name__) # Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request +@cache_for_request_or_task def get_authorized_findings(permission, user=None): impl = get_auth_filter("finding.get_authorized_findings") if impl: @@ -41,7 +41,7 @@ def get_authorized_findings_for_queryset(permission, queryset, user=None): # Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request +@cache_for_request_or_task def get_authorized_vulnerability_ids(permission, user=None): impl = get_auth_filter("finding.get_authorized_vulnerability_ids") if impl: diff --git a/dojo/finding_group/queries.py b/dojo/finding_group/queries.py index c615ca6be61..28002baf4fa 100644 --- a/dojo/finding_group/queries.py +++ b/dojo/finding_group/queries.py @@ -4,11 +4,11 @@ def get_auth_filter(key): return None from dojo.models import Finding_Group -from dojo.request_cache import cache_for_request +from dojo.request_cache import cache_for_request_or_task # Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request +@cache_for_request_or_task def get_authorized_finding_groups(permission, user=None): impl = get_auth_filter("finding_group.get_authorized_finding_groups") if impl: diff --git a/dojo/jira/queries.py b/dojo/jira/queries.py index 418e1fae69d..b762846f58d 100644 --- a/dojo/jira/queries.py +++ b/dojo/jira/queries.py @@ -4,11 +4,11 @@ def get_auth_filter(key): return None from dojo.models import JIRA_Issue, JIRA_Project -from dojo.request_cache import cache_for_request +from dojo.request_cache import cache_for_request_or_task # Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request +@cache_for_request_or_task def get_authorized_jira_projects(permission, user=None): impl = get_auth_filter("jira_link.get_authorized_jira_projects") if impl: @@ -17,7 +17,7 @@ def get_authorized_jira_projects(permission, user=None): # Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request +@cache_for_request_or_task def get_authorized_jira_issues(permission): impl = get_auth_filter("jira_link.get_authorized_jira_issues") if impl: diff --git a/dojo/middleware.py b/dojo/middleware.py index 8f4967fad33..0e158f5e756 100644 --- a/dojo/middleware.py +++ b/dojo/middleware.py @@ -8,16 +8,61 @@ import pghistory.middleware from django.conf import settings from django.db import models +from django.dispatch import receiver from django.http import HttpResponseRedirect from django.urls import reverse from watson.middleware import SearchContextMiddleware from watson.search import search_context_manager +from dojo.caching import ( + cache_dict_to_model, + dojo_settings_cache, + invalidate_dojo_settings_cache, + model_to_cache_dict, + reset_l1_cache, +) from dojo.models import Dojo_User from dojo.product_announcements import LongRunningRequestProductAnnouncement logger = logging.getLogger(__name__) +# In-process (L1) read-through cache for the System_Settings singleton, via the +# shared dojo_settings_cache decorator. Stored as a plain dict (no pickled model +# graph) and rebuilt into a read-only instance per call. Write paths use +# ``objects.get(no_cache=True)`` and are unaffected. +SYSTEM_SETTINGS_CACHE_KEY = "dojo.system_settings.singleton" + + +@dojo_settings_cache(key=SYSTEM_SETTINGS_CACHE_KEY) +def _cached_system_settings_dict(): + from dojo.models import System_Settings # noqa: PLC0415 circular import + settings_obj = System_Settings.objects.get_from_db() + # ``get_from_db`` returns an unsaved defaults instance (pk None) when the row + # can't be read; returning None keeps that out of the cache so the next call + # retries the DB instead of serving stale defaults. + if settings_obj.pk is None: + return None + return model_to_cache_dict(settings_obj) + + +def get_cached_system_settings(): + from dojo.models import System_Settings # noqa: PLC0415 circular import + data = _cached_system_settings_dict() + if not isinstance(data, dict): + return System_Settings() + # read_only=True tags the rebuilt snapshot so System_Settings.save() refuses + # to persist it; writers must use objects.get(no_cache=True) for a fresh row. + return cache_dict_to_model(System_Settings, data, read_only=True) + + +@receiver(models.signals.post_save, sender="dojo.System_Settings") +def _invalidate_system_settings_cache(*args, **kwargs): + # Connected at import time (string sender avoids the circular import) so the + # bust fires in requests, Celery, commands and tests -- not only when a + # middleware instance is constructed. + invalidate_dojo_settings_cache(SYSTEM_SETTINGS_CACHE_KEY) + + EXEMPT_URLS = [re.compile(settings.LOGIN_URL.lstrip("/"))] if hasattr(settings, "LOGIN_EXEMPT_URLS"): EXEMPT_URLS += [re.compile(expr) for expr in settings.LOGIN_EXEMPT_URLS] @@ -74,53 +119,31 @@ def __call__(self, request): return response -class DojoSytemSettingsMiddleware: +class DojoSettingsManagerMiddleware: + # Caching of the System_Settings singleton lives in dojo.caching (L1+L2). This + # middleware only (a) resets the request-scoped L1 tier and (b) surfaces a + # System_Settings DB-read error as a banner. The thread-local carries just that + # error message (set by System_Settings_Manager.get_from_db). _thread_local = local() def __init__(self, get_response): self.get_response = get_response - from dojo.models import System_Settings # noqa: PLC0415 circular import - # Use classmethod directly to avoid keeping reference to middleware instance - models.signals.post_save.connect(DojoSytemSettingsMiddleware.cleanup, sender=System_Settings) def __call__(self, request): - self.load() - try: - # Store error in request for context processor to display - # (We can't use messages here because MessageMiddleware runs after this middleware) - if hasattr(self._thread_local, "system_settings_error"): - request.system_settings_error = self._thread_local.system_settings_error - # Clear from thread-local after copying to request - delattr(self._thread_local, "system_settings_error") - return self.get_response(request) - finally: - # ensure cleanup happens even if an exception occurs - self.cleanup() - - def process_exception(self, request, exception): - self.cleanup() - - @classmethod - def get_system_settings(cls): - if hasattr(cls._thread_local, "system_settings"): - return cls._thread_local.system_settings - return None - - @classmethod - def cleanup(cls, *args, **kwargs): # noqa: ARG003 - if hasattr(cls._thread_local, "system_settings"): - del cls._thread_local.system_settings - if hasattr(cls._thread_local, "system_settings_error"): - delattr(cls._thread_local, "system_settings_error") - - @classmethod - def load(cls): - # cleanup any existing settings first to ensure fresh state - cls.cleanup() - from dojo.models import System_Settings # noqa: PLC0415 circular import - system_settings = System_Settings.objects.get(no_cache=True) - cls._thread_local.system_settings = system_settings - return system_settings + # uwsgi/gunicorn reuse threads across requests, so the in-process L1 cache + # (threading.local) would otherwise persist between requests. Reset it at + # the start of each request so cached singletons are request-scoped. + reset_l1_cache() + # Drop any error left on this reused thread by a previous request. + if hasattr(self._thread_local, "system_settings_error"): + delattr(self._thread_local, "system_settings_error") + # Warm the cache once; this also captures any DB-read error (via + # get_from_db) so the context processor can display it as a banner. + # (We can't use messages here because MessageMiddleware runs after this.) + get_cached_system_settings() + if hasattr(self._thread_local, "system_settings_error"): + request.system_settings_error = self._thread_local.system_settings_error + return self.get_response(request) class System_Settings_Manager(models.Manager): @@ -133,28 +156,31 @@ def get_from_db(self, *args, **kwargs): except Exception as e: # Store error message in thread-local for middleware to display error_msg = str(e) - if hasattr(DojoSytemSettingsMiddleware._thread_local, "system_settings_error"): + if hasattr(DojoSettingsManagerMiddleware._thread_local, "system_settings_error"): # Only store the first error to avoid duplicates pass else: - DojoSytemSettingsMiddleware._thread_local.system_settings_error = error_msg + DojoSettingsManagerMiddleware._thread_local.system_settings_error = error_msg # Return defaults so app can still start - error will be displayed as warning message # logger.debug('unable to get system_settings from database, returning defaults. Exception was:', exc_info=True) return System_Settings() return from_db def get(self, no_cache=False, *args, **kwargs): # noqa: FBT002 - this is bit hard to fix nice have this universally fixed + # NOTE: this override does not behave like Manager.get(). System_Settings is + # a singleton, so the cached path ignores any filter args/kwargs, always + # returns the one row, and never raises DoesNotExist/MultipleObjectsReturned + # (get_or_create against this manager therefore won't create — data + # migrations use apps.get_model(), which keeps Django's default manager). + # The cached instance is a read-only snapshot (see get_cached_system_settings); + # to modify-and-save, fetch a fresh row with no_cache=True. if no_cache: - # logger.debug('no_cache specified or cached value found, loading system settings from db') + # logger.debug('no_cache specified, loading system settings from db') return self.get_from_db(*args, **kwargs) - - from_cache = DojoSytemSettingsMiddleware.get_system_settings() - - if not from_cache: - # logger.debug('no cached value found, loading system settings from db') - return self.get_from_db(*args, **kwargs) - - return from_cache + # Read through the in-process (L1) cache (dojo.caching). L1 is request/task + # scoped (reset by the middleware and the Celery task base), so repeated + # reads within a request are served in-process. + return get_cached_system_settings() class APITrailingSlashMiddleware: diff --git a/dojo/product/queries.py b/dojo/product/queries.py index 44826f046c9..3d24360c58a 100644 --- a/dojo/product/queries.py +++ b/dojo/product/queries.py @@ -11,11 +11,11 @@ def get_auth_filter(key): return None Product, Product_API_Scan_Configuration, ) -from dojo.request_cache import cache_for_request +from dojo.request_cache import cache_for_request_or_task # Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request +@cache_for_request_or_task def get_authorized_products(permission, user=None): impl = get_auth_filter("product.get_authorized_products") if impl: @@ -24,7 +24,7 @@ def get_authorized_products(permission, user=None): # Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request +@cache_for_request_or_task def get_authorized_app_analysis(permission): impl = get_auth_filter("product.get_authorized_app_analysis") if impl: @@ -33,7 +33,7 @@ def get_authorized_app_analysis(permission): # Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request +@cache_for_request_or_task def get_authorized_dojo_meta(permission): impl = get_auth_filter("product.get_authorized_dojo_meta") if impl: @@ -42,7 +42,7 @@ def get_authorized_dojo_meta(permission): # Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request +@cache_for_request_or_task def get_authorized_languages(permission): impl = get_auth_filter("product.get_authorized_languages") if impl: @@ -51,7 +51,7 @@ def get_authorized_languages(permission): # Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request +@cache_for_request_or_task def get_authorized_engagement_presets(permission): impl = get_auth_filter("product.get_authorized_engagement_presets") if impl: @@ -60,7 +60,7 @@ def get_authorized_engagement_presets(permission): # Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request +@cache_for_request_or_task def get_authorized_product_api_scan_configurations(permission): impl = get_auth_filter("product.get_authorized_product_api_scan_configurations") if impl: diff --git a/dojo/product_type/queries.py b/dojo/product_type/queries.py index 16d70d4aad4..b196aa3ff87 100644 --- a/dojo/product_type/queries.py +++ b/dojo/product_type/queries.py @@ -4,11 +4,11 @@ def get_auth_filter(key): return None from dojo.models import Product_Type -from dojo.request_cache import cache_for_request +from dojo.request_cache import cache_for_request_or_task # Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request +@cache_for_request_or_task def get_authorized_product_types(permission): impl = get_auth_filter("product_type.get_authorized_product_types") if impl: diff --git a/dojo/request_cache/__init__.py b/dojo/request_cache/__init__.py index 320f9acd87f..20606136920 100644 --- a/dojo/request_cache/__init__.py +++ b/dojo/request_cache/__init__.py @@ -1,4 +1,6 @@ -from crum import get_current_request +import threading + +from crum import get_current_request, get_current_user # Attribution: This code has been taken from https://github.com/anexia-it/django-request-cache, which has # been published under the MIT License. Since this project hasn't been updated for more than a year, @@ -13,6 +15,44 @@ def get_request_cache(): # noqa: RUF067 return getattr(get_current_request(), "cache", None) +class _TaskCache: # noqa: RUF067 + + """ + Attribute-bag cache with the same access pattern as ``RequestCache`` (values + stored via ``setattr``/``getattr`` on the instance), used inside a Celery task + when there is no request. A fresh instance is installed at each task boundary + (see ``begin_task_cache``), so it can never outlive one task. + """ + + +# Per-thread task cache. Only populated between ``begin_task_cache`` and +# ``end_task_cache`` (called by DojoAsyncTask around each task). ``None`` outside a +# task, so management commands / shells that never hit a task boundary do NOT cache +# (a cache with no reset boundary would grow unbounded and serve stale data). +_task_cache_local = threading.local() # noqa: RUF067 + + +def get_task_cache(): # noqa: RUF067 + """Return the current task's cache, or ``None`` when not inside a task.""" + return getattr(_task_cache_local, "cache", None) + + +def begin_task_cache(): # noqa: RUF067 + """ + Install a fresh task cache. Called at the START of every task so a task never + inherits a cache populated by a prior task on the same (reused) worker thread. + """ + _task_cache_local.cache = _TaskCache() + + +def end_task_cache(): # noqa: RUF067 + """ + Drop the task cache. Called at the END of every task so nothing lingers on the + thread after the task completes. + """ + _task_cache_local.cache = None + + cache_args_kwargs_marker = object() # noqa: RUF067 marker for separating args from kwargs (needs to be global) @@ -58,3 +98,41 @@ def wrapper(*args, **kwargs): return result return wrapper + + +def cache_for_request_or_task(fn): # noqa: RUF067 + """ + Like ``cache_for_request``, but also caches inside a Celery task (which has no + request). Resolves the store as: request cache if a request exists, else the + task cache if inside a task, else no caching (executes directly). + + SECURITY: the cache key folds in the *effective* current user + (``get_current_user().pk``) so that user-dependent results (e.g. authorized + querysets that resolve ``user`` downstream and would otherwise key on + ``user=None``) cannot leak between users. This matters in tasks: a worker + thread is reused across tasks and ``DojoAsyncTask`` impersonates a different + user per task, and a single task can impersonate multiple users (e.g. the + rules engine under ``impersonate(rule.owner)``). The per-task reset + (begin/end) bounds staleness (e.g. revoked permissions); the user-aware key + guarantees isolation even within a task. + """ + def wrapper(*args, **kwargs): + cache = get_request_cache() + if cache is None: + cache = get_task_cache() # populated only inside a DojoAsyncTask + if cache is None: + # neither a request nor a task -> execute without caching + return fn(*args, **kwargs) + + user = get_current_user() + user_pk = user.pk if user is not None else None + key = cache_calculate_key(fn.__name__, user_pk, *args, **kwargs) + + try: + result = getattr(cache, key) + except AttributeError: + result = fn(*args, **kwargs) + setattr(cache, key, result) + + return result + return wrapper diff --git a/dojo/risk_acceptance/queries.py b/dojo/risk_acceptance/queries.py index 495ecc0f985..fb523247d74 100644 --- a/dojo/risk_acceptance/queries.py +++ b/dojo/risk_acceptance/queries.py @@ -4,11 +4,11 @@ def get_auth_filter(key): return None from dojo.models import Risk_Acceptance -from dojo.request_cache import cache_for_request +from dojo.request_cache import cache_for_request_or_task # Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request +@cache_for_request_or_task def get_authorized_risk_acceptances(permission): impl = get_auth_filter("risk_acceptance.get_authorized_risk_acceptances") if impl: diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index 84d5df63887..119663f2302 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -280,6 +280,15 @@ DD_V3_FEATURE_LOCATIONS=(bool, True), # Dictates if v3 org/asset relabeling (+url routing) will be enabled (on by default as of 3.0.0; set to False to restore Product/Product Type labels and URLs) DD_ENABLE_V3_ORGANIZATION_ASSET_RELABEL=(bool, True), + # Shared cache backend (django.core.cache). When set, Django uses RedisCache + # (e.g. redis://valkey:6379/1); when empty it falls back to LocMemCache. Used + # by general framework caching; the singleton settings cache (dojo/caching.py) + # is in-process only and does not read or write this backend. + DD_CACHE_URL=(str, ""), + # In-process (L1) read-through cache for global singleton getters (see + # dojo/caching.py). Per-thread freshness budget in seconds; -1 disables it. + # Reset every request/task, so each request/task reads the singleton once. + DD_SETTINGS_CACHE_L1_TTL=(int, 30), # Notification env-vars (SLA notify, alert refresh/counter/cap, system-level trump). Defined in dojo.notifications.settings. **NOTIFICATIONS_ENV_DEFAULTS, ) @@ -328,6 +337,20 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param # Raises django's ImproperlyConfigured exception if SECRET_KEY not in os.environ SECRET_KEY = env("DD_SECRET_KEY") +# Default cache backend (django.core.cache). Redis when DD_CACHE_URL is set, +# else per-process LocMemCache. General framework caching only; the singleton +# settings cache (dojo/caching.py) is in-process and does not use this backend. +if env("DD_CACHE_URL"): + CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": env("DD_CACHE_URL"), + }, + } + +# In-process singleton cache (dojo/caching.py) +SETTINGS_CACHE_L1_TTL = env("DD_SETTINGS_CACHE_L1_TTL") + # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name # although not all choices may be available on all operating systems. @@ -809,7 +832,7 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param DJANGO_MIDDLEWARE_CLASSES = [ "django.middleware.common.CommonMiddleware", "dojo.middleware.APITrailingSlashMiddleware", - "dojo.middleware.DojoSytemSettingsMiddleware", + "dojo.middleware.DojoSettingsManagerMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.csrf.CsrfViewMiddleware", "django.middleware.security.SecurityMiddleware", diff --git a/dojo/system_settings/models.py b/dojo/system_settings/models.py index 81024a58383..97566256714 100644 --- a/dojo/system_settings/models.py +++ b/dojo/system_settings/models.py @@ -351,6 +351,22 @@ class System_Settings(models.Model): from dojo.middleware import System_Settings_Manager # noqa: PLC0415 circular import objects = System_Settings_Manager() + def save(self, *args, **kwargs): + # Guard against persisting an instance handed back by the read-through + # cache. ``System_Settings.objects.get()`` returns a snapshot rebuilt from + # the cached dict (see dojo.middleware.get_cached_system_settings); saving + # it could overwrite concurrent changes with stale values. Writers must + # fetch a fresh instance with ``System_Settings.objects.get(no_cache=True)``. + from dojo.caching import READ_ONLY_CACHE_MARKER, ReadOnlyCachedInstanceError # noqa: PLC0415 circular import + if getattr(self, READ_ONLY_CACHE_MARKER, False): + msg = ( + "Refusing to save a System_Settings instance obtained from the read-through cache " + "(System_Settings.objects.get()); it is a read-only snapshot. Fetch a fresh instance " + "with System_Settings.objects.get(no_cache=True) before saving." + ) + raise ReadOnlyCachedInstanceError(msg) + super().save(*args, **kwargs) + def clean(self): super().clean() diff --git a/dojo/test/queries.py b/dojo/test/queries.py index 0376cb02dd4..7101037f268 100644 --- a/dojo/test/queries.py +++ b/dojo/test/queries.py @@ -4,11 +4,11 @@ def get_auth_filter(key): return None from dojo.models import Test, Test_Import -from dojo.request_cache import cache_for_request +from dojo.request_cache import cache_for_request_or_task # Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request +@cache_for_request_or_task def get_authorized_tests(permission, product=None): impl = get_auth_filter("test.get_authorized_tests") if impl: @@ -17,7 +17,7 @@ def get_authorized_tests(permission, product=None): # Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request +@cache_for_request_or_task def get_authorized_test_imports(permission): impl = get_auth_filter("test.get_authorized_test_imports") if impl: diff --git a/dojo/tool_product/queries.py b/dojo/tool_product/queries.py index 45dd338b5b3..a2b7740d905 100644 --- a/dojo/tool_product/queries.py +++ b/dojo/tool_product/queries.py @@ -3,12 +3,12 @@ except ImportError: def get_auth_filter(key): return None -from dojo.request_cache import cache_for_request +from dojo.request_cache import cache_for_request_or_task from dojo.tool_product.models import Tool_Product_Settings # Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request +@cache_for_request_or_task def get_authorized_tool_product_settings(permission): impl = get_auth_filter("tool_product.get_authorized_tool_product_settings") if impl: diff --git a/dojo/user/queries.py b/dojo/user/queries.py index 136bfcd9c67..611e44ed5fe 100644 --- a/dojo/user/queries.py +++ b/dojo/user/queries.py @@ -6,7 +6,7 @@ def get_auth_filter(key): return None from dojo.models import ( Dojo_User, ) -from dojo.request_cache import cache_for_request +from dojo.request_cache import cache_for_request_or_task def get_authorized_users_for_product_type(users, product_type, permission): @@ -24,7 +24,7 @@ def get_authorized_users_for_product_and_product_type(users, product, permission # Cached because it is a complex SQL query and it is called 3 times for the engagement lists in products -@cache_for_request +@cache_for_request_or_task def get_authorized_users(permission, user=None): impl = get_auth_filter("user.get_authorized_users") if impl: diff --git a/unittests/dojo_test_case.py b/unittests/dojo_test_case.py index 37ca212c412..b6d6b0d864b 100644 --- a/unittests/dojo_test_case.py +++ b/unittests/dojo_test_case.py @@ -16,12 +16,13 @@ from rest_framework.test import APIClient, APITestCase from vcr_unittest import VCRTestCase +from dojo.caching import invalidate_dojo_settings_cache from dojo.importers.location_manager import LocationManager from dojo.jira import helper as jira_helper from dojo.jira.views import get_custom_field from dojo.location.models import Location, LocationFindingReference from dojo.location.status import FindingLocationStatus -from dojo.middleware import DojoSytemSettingsMiddleware +from dojo.middleware import SYSTEM_SETTINGS_CACHE_KEY, get_cached_system_settings from dojo.models import ( SEVERITIES, DojoMeta, @@ -44,6 +45,17 @@ logger = logging.getLogger(__name__) +def refresh_system_settings_cache(): + """ + Make the next ``System_Settings.objects.get()`` reflect DB changes made in a + test. Tests mutate settings via ``.update()`` (which fires no post_save) so + the L1/L2 cache isn't auto-busted; drop it and re-warm so a subsequent cached + read is served in-process. Replaces the old middleware ``load()``. + """ + invalidate_dojo_settings_cache(SYSTEM_SETTINGS_CACHE_KEY) + get_cached_system_settings() + + def get_unit_tests_path(): return Path(__file__).parent @@ -61,14 +73,14 @@ def wrapper(*args, **kwargs): # Set the flag to the specified value System_Settings.objects.update(**{flag_name: value}) # Reinitialize middleware with updated settings as this doesn't happen automatically during django tests - DojoSytemSettingsMiddleware.load() + refresh_system_settings_cache() try: return test_func(*args, **kwargs) finally: # Reset the flag to its original state after the test System_Settings.objects.update(**{flag_name: not value}) # Reinitialize middleware with updated settings as this doesn't happen automatically during django tests - DojoSytemSettingsMiddleware.load() + refresh_system_settings_cache() return wrapper return decorator @@ -84,14 +96,14 @@ def wrapper(*args, **kwargs): # Set the flag to the specified value System_Settings.objects.update(**{field: value}) # Reinitialize middleware with updated settings as this doesn't happen automatically during django tests - DojoSytemSettingsMiddleware.load() + refresh_system_settings_cache() try: return test_func(*args, **kwargs) finally: # Reset the flag to its original state after the test System_Settings.objects.update(**{field: old_value}) # Reinitialize middleware with updated settings as this doesn't happen automatically during django tests - DojoSytemSettingsMiddleware.load() + refresh_system_settings_cache() return wrapper @@ -126,12 +138,12 @@ def get_test_admin(self, *args, **kwargs): return User.objects.get(username="admin") def system_settings(self, **kwargs): - ss = System_Settings.objects.get() + ss = System_Settings.objects.get(no_cache=True) for key, value in kwargs.items(): setattr(ss, key, value) ss.save() # Refresh the cache with the new settings - DojoSytemSettingsMiddleware.load() + refresh_system_settings_cache() def create_product_type(self, name, *args, description="dummy description", **kwargs): product_type = Product_Type(name=name, description=description) @@ -566,7 +578,7 @@ def __init__(self, *args, **kwargs): def setUp(self): super().setUp() # Initialize middleware with fresh settings from db - DojoSytemSettingsMiddleware.load() + refresh_system_settings_cache() def common_check_finding(self, finding): self.assertIn(finding.severity, SEVERITIES) diff --git a/unittests/test_caching.py b/unittests/test_caching.py new file mode 100644 index 00000000000..a1e8c1487c1 --- /dev/null +++ b/unittests/test_caching.py @@ -0,0 +1,181 @@ +from crum import get_current_user, impersonate +from django.test import TestCase, override_settings + +from dojo.caching import ( + _L1_STORE, # noqa: PLC2701 test needs the in-process store + READ_ONLY_CACHE_MARKER, + ReadOnlyCachedInstanceError, + cache_dict_to_model, + dojo_settings_cache, + invalidate_dojo_settings_cache, + model_to_cache_dict, +) +from dojo.models import Dojo_User, System_Settings +from dojo.request_cache import begin_task_cache, cache_for_request_or_task, end_task_cache + +from .dojo_test_case import DojoTestCase + + +@override_settings(SETTINGS_CACHE_L1_TTL=30) +class DojoSettingsCacheTest(TestCase): + + """ + Unit tests for the in-process (L1) read-through decorator (dojo/caching.py). + + There is no shared/L2 tier: the decorator only memoizes in-process and relies + on L1 reset at request/task boundaries for cross-process freshness. + """ + + def setUp(self): + _L1_STORE.clear() + + def tearDown(self): + _L1_STORE.clear() + + def _build_getter(self, *, key="k", returns=1): + calls = {"n": 0} + + @dojo_settings_cache(key=key) + def getter(): + calls["n"] += 1 + return returns + + return getter, calls + + def test_l1_memoizes_in_process(self): + getter, calls = self._build_getter() + getter() + getter() + getter() + self.assertEqual(calls["n"], 1) # computed once, then served from L1 + + @override_settings(SETTINGS_CACHE_L1_TTL=-1) + def test_l1_disabled_is_passthrough(self): + getter, calls = self._build_getter() + getter() + getter() + self.assertEqual(calls["n"], 2) # recomputed every call when L1 off + + def test_reset_recomputes(self): + getter, calls = self._build_getter() + getter() + _L1_STORE.reset() + getter() + self.assertEqual(calls["n"], 2) # reset drops L1, so it recomputes + + def test_none_result_is_not_cached(self): + getter, calls = self._build_getter(returns=None) + self.assertIsNone(getter()) + self.assertIsNone(getter()) + self.assertEqual(calls["n"], 2) # None recomputed each call (never stored) + + def test_invalidate_recomputes(self): + getter, calls = self._build_getter(returns=7) + self.assertEqual(getter(), 7) + invalidate_dojo_settings_cache("k") + getter() + self.assertEqual(calls["n"], 2) # recomputed after invalidation + + +class ModelDictRoundTripTest(DojoTestCase): + + def test_round_trip_preserves_field_values(self): + settings_obj = System_Settings.objects.get(no_cache=True) + data = model_to_cache_dict(settings_obj) + self.assertIsInstance(data, dict) + self.assertEqual(data["id"], settings_obj.pk) + rebuilt = cache_dict_to_model(System_Settings, data) + self.assertEqual(rebuilt.pk, settings_obj.pk) + self.assertEqual(rebuilt.enable_deduplication, settings_obj.enable_deduplication) + + def test_read_only_flag_only_when_requested(self): + data = model_to_cache_dict(System_Settings.objects.get(no_cache=True)) + # default: saveable (no marker) + self.assertFalse(getattr(cache_dict_to_model(System_Settings, data), READ_ONLY_CACHE_MARKER, False)) + # read_only=True: tagged as a read-only snapshot + self.assertTrue(getattr(cache_dict_to_model(System_Settings, data, read_only=True), READ_ONLY_CACHE_MARKER, False)) + + +class SystemSettingsSaveGuardTest(DojoTestCase): + + """ + The read-through cache hands back a read-only snapshot: saving the instance + returned by ``System_Settings.objects.get()`` must fail loudly, while a fresh + ``no_cache=True`` instance saves normally. + """ + + def setUp(self): + _L1_STORE.clear() + + def tearDown(self): + _L1_STORE.clear() + + def test_saving_cached_instance_raises(self): + cached = System_Settings.objects.get() # read-through (cached) path + self.assertTrue(getattr(cached, READ_ONLY_CACHE_MARKER, False)) + cached.enable_deduplication = not cached.enable_deduplication + with self.assertRaises(ReadOnlyCachedInstanceError): + cached.save() + + def test_no_cache_instance_is_saveable(self): + fresh = System_Settings.objects.get(no_cache=True) + self.assertFalse(getattr(fresh, READ_ONLY_CACHE_MARKER, False)) + fresh.enable_deduplication = not fresh.enable_deduplication + fresh.save() # must not raise + + +class CacheForRequestOrTaskTest(DojoTestCase): + + """ + ``cache_for_request_or_task`` caches within a task (no request) and isolates by + the effective (impersonated) user, so per-user results cannot leak between users + on a reused worker thread. + """ + + def setUp(self): + self.calls = {"n": 0} + + @cache_for_request_or_task + def getter(): + self.calls["n"] += 1 + user = get_current_user() + return (self.calls["n"], user.pk if user else None) + + self.getter = getter + self.user_a = Dojo_User.objects.create(username="roq_user_a") + self.user_b = Dojo_User.objects.create(username="roq_user_b") + + def tearDown(self): + end_task_cache() # ensure no task cache leaks to the next test + + def test_no_context_is_passthrough(self): + self.getter() + self.getter() + self.assertEqual(self.calls["n"], 2) # no request and no task -> recomputed + + def test_task_scope_caches(self): + begin_task_cache() + with impersonate(self.user_a): + r1 = self.getter() + r2 = self.getter() + self.assertEqual(r1, r2) + self.assertEqual(self.calls["n"], 1) # served from the task cache + + def test_task_scope_isolates_users(self): + begin_task_cache() + with impersonate(self.user_a): + a = self.getter() + with impersonate(self.user_b): + b = self.getter() # same task, different user -> must NOT get a's value + self.assertEqual(a[1], self.user_a.pk) + self.assertEqual(b[1], self.user_b.pk) + self.assertNotEqual(a, b) + + def test_end_task_cache_clears(self): + begin_task_cache() + with impersonate(self.user_a): + self.getter() + end_task_cache() + with impersonate(self.user_a): + self.getter() # no task cache and no request -> recomputed + self.assertEqual(self.calls["n"], 2) diff --git a/unittests/test_deduplication_logic.py b/unittests/test_deduplication_logic.py index fd6b5d2847c..e4de8522d92 100644 --- a/unittests/test_deduplication_logic.py +++ b/unittests/test_deduplication_logic.py @@ -1894,7 +1894,7 @@ def create_new_test_and_engagment_from_finding(self, finding): return test_new, eng_new def enable_dedupe(self, *, enable=True): - system_settings = System_Settings.objects.get() + system_settings = System_Settings.objects.get(no_cache=True) system_settings.enable_deduplication = enable system_settings.save() diff --git a/unittests/test_duplication_loops.py b/unittests/test_duplication_loops.py index a3a70cf5c2b..3246343d533 100644 --- a/unittests/test_duplication_loops.py +++ b/unittests/test_duplication_loops.py @@ -378,7 +378,7 @@ def test_list_relations_for_three_reverse(self): # Test that Delete Duplicate Findings & Maximum Duplicate is correctly deleting olding finding first based off of finding date value def test_delete_duplicate_order(self): # Turn on delete duplicates and set the maximum dedupe value to 1 - system_settings = System_Settings.objects.get() + system_settings = System_Settings.objects.get(no_cache=True) system_settings.delete_duplicates = True system_settings.max_dupes = 1 system_settings.save() @@ -409,7 +409,7 @@ def test_delete_duplicate_order(self): def test_delete_duplicate_order_same_date_tiebreak_by_id(self): """When duplicate findings share the same date, excess deletes use id as tie-break (oldest id first).""" - system_settings = System_Settings.objects.get() + system_settings = System_Settings.objects.get(no_cache=True) system_settings.delete_duplicates = True system_settings.max_dupes = 1 system_settings.save() diff --git a/unittests/test_false_positive_history_logic.py b/unittests/test_false_positive_history_logic.py index 5975348e14e..0bc7e58362f 100644 --- a/unittests/test_false_positive_history_logic.py +++ b/unittests/test_false_positive_history_logic.py @@ -4,6 +4,7 @@ from crum import impersonate from django.conf import settings +from django.test import override_settings from dojo.finding.deduplication import do_false_positive_history_batch from dojo.finding.ui.views import EditFinding @@ -126,6 +127,7 @@ @versioned_fixtures +@override_settings(SETTINGS_CACHE_L1_TTL=30, SETTINGS_CACHE_L2_TTL=-1) class TestFalsePositiveHistoryLogic(DojoTestCase): fixtures = ["dojo_testdata.json"] @@ -1684,7 +1686,7 @@ def test_fp_history_batch_issues_single_candidate_query(self): # 4 lazy-load chain: findings[0].test / .engagement / .product / .test_type # 1 candidates SELECT (with .only()) # 1 bulk UPDATE - with self.assertNumQueries(7): + with self.assertNumQueries(6): do_false_positive_history_batch(batch) # One candidate-fetch call for the whole batch — not one per finding. self.assertEqual(mock_fetch.call_count, 1, "Expected exactly one call to _fetch_fp_candidates_for_batch") @@ -1713,7 +1715,7 @@ def test_fp_history_batch_retroactive_marks_existing_active_fp(self): # 4 lazy-load chain: findings[0].test / .engagement / .product / .test_type # 1 candidates SELECT (with .only()) # 1 bulk UPDATE - with self.assertNumQueries(7): + with self.assertNumQueries(6): do_false_positive_history_batch(batch) # The pre-existing active finding must now be retroactively marked FP. @@ -1721,9 +1723,9 @@ def test_fp_history_batch_retroactive_marks_existing_active_fp(self): def test_fp_history_batch_query_count_does_not_grow_with_affected_findings(self): """ - Query count must stay flat (7) no matter how many findings are retroactively marked. + Query count must stay flat (6) no matter how many findings are retroactively marked. - With the old per-finding approach this would have been 7 + N queries where N is the + With the old per-finding approach this would have been 6 + N queries where N is the number of pre-existing findings that get marked as FP. With the batch approach it is always 7: System_Settings, 4 lazy-load chain, candidates SELECT, one bulk UPDATE. """ @@ -1748,7 +1750,7 @@ def test_fp_history_batch_query_count_does_not_grow_with_affected_findings(self) # 4 lazy-load chain: findings[0].test / .engagement / .product / .test_type # 1 candidates SELECT (with .only()) # 1 bulk UPDATE covering all retroactively marked findings - with self.assertNumQueries(7): + with self.assertNumQueries(6): do_false_positive_history_batch(batch) # All pre-existing findings must now be marked as FP. @@ -2029,26 +2031,26 @@ def create_new_test_and_engagment_and_product_from_finding(self, finding): return test_new, eng_new, product_new def enable_false_positive_history(self): - system_settings = System_Settings.objects.get() + system_settings = System_Settings.objects.get(no_cache=True) system_settings.false_positive_history = True system_settings.save() def enable_retroactive_false_positive_history(self): - system_settings = System_Settings.objects.get() + system_settings = System_Settings.objects.get(no_cache=True) system_settings.retroactive_false_positive_history = True system_settings.save() def disable_retroactive_false_positive_history(self): - system_settings = System_Settings.objects.get() + system_settings = System_Settings.objects.get(no_cache=True) system_settings.retroactive_false_positive_history = False system_settings.save() def enable_dedupe(self): - system_settings = System_Settings.objects.get() + system_settings = System_Settings.objects.get(no_cache=True) system_settings.enable_deduplication = True system_settings.save() def disable_dedupe(self): - system_settings = System_Settings.objects.get() + system_settings = System_Settings.objects.get(no_cache=True) system_settings.enable_deduplication = False system_settings.save() diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index e425743a9fa..14fea501e17 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -275,7 +275,7 @@ def _import_reimport_performance( @tag("performance") -@override_settings(V3_FEATURE_LOCATIONS=False) +@override_settings(V3_FEATURE_LOCATIONS=False, SETTINGS_CACHE_L1_TTL=30, SETTINGS_CACHE_L2_TTL=-1) class TestDojoImporterPerformanceSmall(TestDojoImporterPerformanceBase): """Performance tests using small sample files (StackHawk, ~6 findings).""" @@ -349,7 +349,7 @@ def test_import_reimport_reimport_performance_pghistory_async(self): expected_num_async_tasks2=1, expected_num_queries3=30, expected_num_async_tasks3=1, - expected_num_queries4=100, + expected_num_queries4=106, expected_num_async_tasks4=0, ) @@ -367,13 +367,13 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._import_reimport_performance( - expected_num_queries1=174, + expected_num_queries1=176, expected_num_async_tasks1=2, - expected_num_queries2=133, + expected_num_queries2=134, expected_num_async_tasks2=1, - expected_num_queries3=39, + expected_num_queries3=40, expected_num_async_tasks3=1, - expected_num_queries4=100, + expected_num_queries4=106, expected_num_async_tasks4=0, ) @@ -392,13 +392,13 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr self.system_settings(enable_product_grade=True) self._import_reimport_performance( - expected_num_queries1=182, + expected_num_queries1=186, expected_num_async_tasks1=5, - expected_num_queries2=141, + expected_num_queries2=144, expected_num_async_tasks2=4, - expected_num_queries3=46, + expected_num_queries3=49, expected_num_async_tasks3=3, - expected_num_queries4=107, + expected_num_queries4=115, expected_num_async_tasks4=3, ) @@ -547,9 +547,9 @@ def test_deduplication_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._deduplication_performance( - expected_num_queries1=110, + expected_num_queries1=112, expected_num_async_tasks1=2, - expected_num_queries2=91, + expected_num_queries2=93, expected_num_async_tasks2=2, ) @@ -590,7 +590,7 @@ def test_deduplication_performance_pghistory_async_wait(self): @tag("performance") -@override_settings(V3_FEATURE_LOCATIONS=True) +@override_settings(V3_FEATURE_LOCATIONS=True, SETTINGS_CACHE_L1_TTL=30, SETTINGS_CACHE_L2_TTL=-1) class TestDojoImporterPerformanceSmallLocations(TestDojoImporterPerformanceBase): r""" @@ -676,7 +676,7 @@ def test_import_reimport_reimport_performance_pghistory_async(self): expected_num_async_tasks2=1, expected_num_queries3=38, expected_num_async_tasks3=1, - expected_num_queries4=101, + expected_num_queries4=107, expected_num_async_tasks4=0, ) @@ -694,13 +694,13 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._import_reimport_performance( - expected_num_queries1=183, + expected_num_queries1=185, expected_num_async_tasks1=2, - expected_num_queries2=144, + expected_num_queries2=145, expected_num_async_tasks2=1, - expected_num_queries3=49, + expected_num_queries3=50, expected_num_async_tasks3=1, - expected_num_queries4=101, + expected_num_queries4=107, expected_num_async_tasks4=0, ) @@ -719,13 +719,13 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr self.system_settings(enable_product_grade=True) self._import_reimport_performance( - expected_num_queries1=194, + expected_num_queries1=198, expected_num_async_tasks1=5, - expected_num_queries2=155, + expected_num_queries2=158, expected_num_async_tasks2=4, - expected_num_queries3=56, + expected_num_queries3=59, expected_num_async_tasks3=3, - expected_num_queries4=111, + expected_num_queries4=119, expected_num_async_tasks4=3, ) @@ -846,8 +846,8 @@ def test_deduplication_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._deduplication_performance( - expected_num_queries1=119, + expected_num_queries1=121, expected_num_async_tasks1=2, - expected_num_queries2=202, + expected_num_queries2=204, expected_num_async_tasks2=2, ) diff --git a/unittests/test_metrics_queries.py b/unittests/test_metrics_queries.py index 4a0cf3c87eb..94ec80f2035 100644 --- a/unittests/test_metrics_queries.py +++ b/unittests/test_metrics_queries.py @@ -4,7 +4,7 @@ from datetime import date, datetime from unittest.mock import patch -from django.test import RequestFactory +from django.test import RequestFactory, override_settings from django.urls import reverse from dojo.metrics import utils @@ -47,6 +47,7 @@ def add(*args, **kwargs): @versioned_fixtures +@override_settings(SETTINGS_CACHE_L1_TTL=30, SETTINGS_CACHE_L2_TTL=-1) class FindingQueriesTest(DojoTestCase): fixtures = ["dojo_testdata.json", "unit_metrics_additional_data.json"] @@ -90,7 +91,7 @@ def test_finding_queries(self, mock_timezone): # Queries over Finding (legacy auth: fewer auth-layer queries # than RBAC since per-action role-permission lookups are gone). - with self.assertNumQueries(27): + with self.assertNumQueries(25): product_types = [] finding_queries = utils.finding_queries( product_types, @@ -267,6 +268,7 @@ def test_closed_findings_filtered_by_mitigated_date(self, mock_timezone): # TODO: Delete this after the move to Locations @skip_unless_v2 +@override_settings(SETTINGS_CACHE_L1_TTL=30, SETTINGS_CACHE_L2_TTL=-1) class EndpointQueriesTest(DojoTestCase): fixtures = ["dojo_testdata.json"] @@ -302,7 +304,7 @@ def test_endpoint_queries(self, mock_now): # Queries over Finding and Endpoint_Status (legacy auth: fewer # auth-layer queries than RBAC). - with self.assertNumQueries(41): + with self.assertNumQueries(40): product_types = Product_Type.objects.all() endpoint_queries = utils.endpoint_queries( product_types, diff --git a/unittests/test_notifications.py b/unittests/test_notifications.py index 820aae23f07..ad788b57415 100644 --- a/unittests/test_notifications.py +++ b/unittests/test_notifications.py @@ -943,7 +943,7 @@ def test_system_webhook_timeout(self): self.sys_wh.url = f"{self.url_base}/delay/3" self.sys_wh.save() - system_settings = System_Settings.objects.get() + system_settings = System_Settings.objects.get(no_cache=True) system_settings.webhooks_notifications_timeout = 1 system_settings.save() diff --git a/unittests/test_risk_acceptance.py b/unittests/test_risk_acceptance.py index 628624b6db1..de568f037c9 100644 --- a/unittests/test_risk_acceptance.py +++ b/unittests/test_risk_acceptance.py @@ -252,7 +252,7 @@ def create_multiple_ras(self): def test_expiration_handler(self): ra1, ra2, ra3 = self.create_multiple_ras() - system_settings = System_Settings.objects.get() + system_settings = System_Settings.objects.get(no_cache=True) system_settings.risk_acceptance_notify_before_expiration = 10 system_settings.save() heads_up_days = system_settings.risk_acceptance_notify_before_expiration diff --git a/unittests/test_system_settings.py b/unittests/test_system_settings.py index e127bec8f59..f37dfeccb33 100644 --- a/unittests/test_system_settings.py +++ b/unittests/test_system_settings.py @@ -1,12 +1,12 @@ -from unittest.mock import Mock +from unittest import mock -from django.db import models from django.http import HttpResponse from django.test import RequestFactory, TestCase, override_settings from django.urls import reverse from django.utils.timezone import now -from dojo.middleware import DojoSytemSettingsMiddleware +from dojo.caching import invalidate_dojo_settings_cache, reset_l1_cache +from dojo.middleware import SYSTEM_SETTINGS_CACHE_KEY, DojoSettingsManagerMiddleware, get_cached_system_settings from dojo.models import ( Engagement, Finding, @@ -26,18 +26,18 @@ class TestSystemSettings(DojoTestCase): def test_system_settings_update(self): try: # although the unittests are run after initial data has been loaded, for some reason in cicd sometimes the settings aren't present - system_settings = System_Settings.objects.get() + system_settings = System_Settings.objects.get(no_cache=True) except System_Settings.DoesNotExist: system_settings = System_Settings() system_settings.enable_jira = True system_settings.save() - system_settings = System_Settings.objects.get() + system_settings = System_Settings.objects.get(no_cache=True) self.assertEqual(system_settings.enable_jira, True) system_settings.enable_jira = False system_settings.save() - system_settings = System_Settings.objects.get() + system_settings = System_Settings.objects.get(no_cache=True) self.assertEqual(system_settings.enable_jira, False) system_settings.enable_jira = True @@ -93,185 +93,72 @@ def test_post_request_initializes_form_with_finding_instance(self): self.assertIn(response.status_code, [200, 302]) +@override_settings(SETTINGS_CACHE_L1_TTL=30) class TestSystemSettingsMiddlewareIntegration(DojoTestCase): - """Integration tests for DojoSytemSettingsMiddleware using RequestFactory.""" + """ + Integration tests for DojoSettingsManagerMiddleware + System_Settings_Manager. + + Caching lives in dojo.caching (in-process L1 decorator); the middleware resets + the request-scoped L1 tier and surfaces a load error. These tests pin L1 on via + override_settings so they don't depend on the compose env. + """ def setUp(self): - """Set up test environment.""" super().setUp() self.factory = RequestFactory() - # Ensure signal is connected - models.signals.post_save.disconnect(DojoSytemSettingsMiddleware.cleanup, sender=System_Settings) - models.signals.post_save.connect(DojoSytemSettingsMiddleware.cleanup, sender=System_Settings) - - def test_middleware_loads_cache_on_request(self): - """Test that middleware loads settings into cache when processing a request.""" - # Ensure cache is empty - DojoSytemSettingsMiddleware.cleanup() - self.assertIsNone(DojoSytemSettingsMiddleware.get_system_settings()) - - # Create middleware with mock get_response - mock_response = HttpResponse("OK") - mock_get_response = Mock(return_value=mock_response) - middleware = DojoSytemSettingsMiddleware(mock_get_response) - - # Create a request - request = self.factory.get("/test/") - - # Process request through middleware - response = middleware(request) - - # Verify response is returned - self.assertEqual(response, mock_response) - mock_get_response.assert_called_once_with(request) - - # Verify cache was populated during request processing - # Note: cache should be cleaned up after request, but we can check during processing - # Since cleanup happens in finally block, cache should be empty after __call__ returns - self.assertIsNone(DojoSytemSettingsMiddleware.get_system_settings()) - - def test_middleware_cleans_up_cache_after_request(self): - """Test that middleware cleans up cache after request processing.""" - # Manually load cache first - DojoSytemSettingsMiddleware.load() - self.assertIsNotNone(DojoSytemSettingsMiddleware.get_system_settings()) - - # Create middleware - middleware = DojoSytemSettingsMiddleware(lambda _r: HttpResponse("OK")) - - # Process request - request = self.factory.get("/test/") - middleware(request) - - # Verify cache is cleaned up after request - self.assertIsNone(DojoSytemSettingsMiddleware.get_system_settings()) - - def test_middleware_cleans_up_cache_on_exception(self): - """Test that middleware cleans up cache even when exception occurs.""" - # Load cache first - DojoSytemSettingsMiddleware.load() - self.assertIsNotNone(DojoSytemSettingsMiddleware.get_system_settings()) - - # Create middleware that raises an exception - def failing_get_response(request): - msg = "Test exception" - raise ValueError(msg) - - middleware = DojoSytemSettingsMiddleware(failing_get_response) - - # Process request - should raise exception - request = self.factory.get("/test/") - with self.assertRaises(ValueError): - middleware(request) - - # Verify cache is cleaned up even after exception - self.assertIsNone(DojoSytemSettingsMiddleware.get_system_settings()) - - def test_middleware_process_exception_cleans_up_cache(self): - """Test that process_exception method cleans up cache.""" - # Load cache first - DojoSytemSettingsMiddleware.load() - self.assertIsNotNone(DojoSytemSettingsMiddleware.get_system_settings()) - # Create middleware - middleware = DojoSytemSettingsMiddleware(lambda _r: HttpResponse("OK")) + def test_no_cache_always_hits_db(self): + # no_cache bypasses both tiers: every read is a fresh query. + with self.assertNumQueries(2): + System_Settings.objects.get(no_cache=True) + System_Settings.objects.get(no_cache=True) - # Call process_exception directly - request = self.factory.get("/test/") - exception = ValueError("Test exception") - middleware.process_exception(request, exception) - - # Verify cache is cleaned up - self.assertIsNone(DojoSytemSettingsMiddleware.get_system_settings()) - - def test_middleware_cache_isolation_between_requests(self): - """Test that cache is isolated between requests (thread-local).""" - # Create middleware - middleware = DojoSytemSettingsMiddleware(lambda _r: HttpResponse("OK")) - - # First request - request1 = self.factory.get("/test1/") - middleware(request1) - self.assertIsNone(DojoSytemSettingsMiddleware.get_system_settings()) - - # Second request - cache should be empty at start - request2 = self.factory.get("/test2/") - middleware(request2) - self.assertIsNone(DojoSytemSettingsMiddleware.get_system_settings()) - - def test_middleware_cache_during_request_processing(self): - """Test that cache is available during request processing.""" - # Track if cache was available during request - cache_available_during_request = [] - - def get_response_with_cache_check(request): - # Check if cache is available during request processing - cached = DojoSytemSettingsMiddleware.get_system_settings() - cache_available_during_request.append(cached is not None) - return HttpResponse("OK") - - middleware = DojoSytemSettingsMiddleware(get_response_with_cache_check) - - # Process request - request = self.factory.get("/test/") - middleware(request) - - # Verify cache was available during request processing - self.assertTrue(cache_available_during_request[0], "Cache should be available during request processing") - - # But cleaned up after request - self.assertIsNone(DojoSytemSettingsMiddleware.get_system_settings()) - - def test_multiple_get_calls_use_cache(self): - """Test that multiple calls to System_Settings.objects.get() use cache instead of multiple DB queries.""" - # Ensure cache is empty - DojoSytemSettingsMiddleware.cleanup() - - # First call should hit DB (cache is empty) + def test_repeated_cached_get_served_from_l1(self): + # Cold start, warm once (1 query), then repeated cached reads do no queries. + reset_l1_cache() + invalidate_dojo_settings_cache(SYSTEM_SETTINGS_CACHE_KEY) with self.assertNumQueries(1): - settings1 = System_Settings.objects.get() - - # Load into cache via middleware - DojoSytemSettingsMiddleware.load() - - # Now multiple calls should use cache (no additional DB queries) + get_cached_system_settings() with self.assertNumQueries(0): - settings2 = System_Settings.objects.get() - settings3 = System_Settings.objects.get() - settings4 = System_Settings.objects.get() + s2 = System_Settings.objects.get() + s3 = System_Settings.objects.get() + # Rebuilt per call from the cached dict: equal data, not the same instance. + self.assertEqual(s2.pk, s3.pk) + + def test_middleware_resets_l1_each_request(self): + # Warm L1, change the row underneath via update() (no post_save signal, so + # L1 is not auto-busted), then a request must still see the new value + # because the middleware resets L1 at request start. + get_cached_system_settings() + System_Settings.objects.update(enable_deduplication=True) + seen = [] + + def view(_request): + seen.append(System_Settings.objects.get().enable_deduplication) + return HttpResponse("OK") - # All calls should return the same cached object instance - self.assertEqual(settings1.id, settings2.id) - self.assertEqual(settings2.id, settings3.id) - self.assertEqual(settings3.id, settings4.id) - # Verify they're the same object instance (same memory address) - self.assertIs(settings2, settings3) - self.assertIs(settings3, settings4) + middleware = DojoSettingsManagerMiddleware(view) + middleware(self.factory.get("/test/")) + self.assertEqual(seen, [True]) - def test_multiple_get_calls_within_request_use_cache(self): - """Test that multiple get() calls within a single request use cache.""" - retrieved_settings = [] + def test_middleware_surfaces_load_error(self): + # When the DB read fails, get_from_db stashes an error on the thread-local + # and returns defaults; the middleware copies the error onto the request + # for the banner context processor. + invalidate_dojo_settings_cache(SYSTEM_SETTINGS_CACHE_KEY) + reset_l1_cache() + captured = {} - def get_response_with_multiple_gets(request): - # Make multiple calls to get() during request processing - retrieved_settings.append(System_Settings.objects.get()) - retrieved_settings.append(System_Settings.objects.get()) - retrieved_settings.append(System_Settings.objects.get()) + def view(request): + captured["err"] = getattr(request, "system_settings_error", None) return HttpResponse("OK") - middleware = DojoSytemSettingsMiddleware(get_response_with_multiple_gets) - - # Process request - should only hit DB once (when loading cache) - # Then all subsequent get() calls should use cache - request = self.factory.get("/test/") - with self.assertNumQueries(1): # Only one query to load settings into cache - middleware(request) - - # Verify we got 3 settings objects - self.assertEqual(len(retrieved_settings), 3) + def failing_get_from_db(*args, **kwargs): + DojoSettingsManagerMiddleware._thread_local.system_settings_error = "boom" + return System_Settings() # defaults (pk None) -> not cached - # All should be the same cached instance - self.assertIs(retrieved_settings[0], retrieved_settings[1]) - self.assertIs(retrieved_settings[1], retrieved_settings[2]) - self.assertEqual(retrieved_settings[0].id, retrieved_settings[1].id) + middleware = DojoSettingsManagerMiddleware(view) + with mock.patch.object(System_Settings.objects, "get_from_db", side_effect=failing_get_from_db): + middleware(self.factory.get("/test/")) + self.assertEqual(captured["err"], "boom") diff --git a/unittests/test_tag_inheritance_perf.py b/unittests/test_tag_inheritance_perf.py index 26d80be66de..5b3f07e8b59 100644 --- a/unittests/test_tag_inheritance_perf.py +++ b/unittests/test_tag_inheritance_perf.py @@ -99,6 +99,8 @@ def _make_locations(product: Product, n: int) -> None: @override_settings( CELERY_TASK_ALWAYS_EAGER=True, CELERY_TASK_EAGER_PROPAGATES=True, + SETTINGS_CACHE_L1_TTL=30, + SETTINGS_CACHE_L2_TTL=-1, ) class TagInheritancePerfBaselines(DojoTestCase): @@ -135,7 +137,7 @@ def setUpTestData(cls): # Per-product flag is also set in _make_product_with_findings as a # belt-and-braces measure (tests should not be flag-coupled). from dojo.models import System_Settings # noqa: PLC0415 - ss = System_Settings.objects.get() + ss = System_Settings.objects.get(no_cache=True) ss.enable_product_tag_inheritance = True ss.save() @@ -405,10 +407,10 @@ def test_baseline_product_tag_remove_propagates_to_100_locations_v3(self): EXPECTED_PRODUCT_TAG_REMOVE_100_V2 = 53 EXPECTED_PRODUCT_TAG_REMOVE_100_V3 = 53 - EXPECTED_CREATE_ONE_FINDING_V2 = 55 - EXPECTED_CREATE_ONE_FINDING_V3 = 55 - EXPECTED_CREATE_100_FINDINGS_V2 = 3124 - EXPECTED_CREATE_100_FINDINGS_V3 = 3124 + EXPECTED_CREATE_ONE_FINDING_V2 = 56 + EXPECTED_CREATE_ONE_FINDING_V3 = 56 + EXPECTED_CREATE_100_FINDINGS_V2 = 3224 + EXPECTED_CREATE_100_FINDINGS_V3 = 3224 EXPECTED_FINDING_ADD_USER_TAG_V2 = 17 EXPECTED_FINDING_ADD_USER_TAG_V3 = 17 @@ -433,6 +435,8 @@ def test_baseline_product_tag_remove_propagates_to_100_locations_v3(self): CELERY_TASK_ALWAYS_EAGER=True, CELERY_TASK_EAGER_PROPAGATES=True, SECURE_SSL_REDIRECT=False, + SETTINGS_CACHE_L1_TTL=30, + SETTINGS_CACHE_L2_TTL=-1, ) @versioned_fixtures class TagInheritanceImportPerfBaselines(DojoAPITestCase): @@ -457,7 +461,7 @@ class TagInheritanceImportPerfBaselines(DojoAPITestCase): @classmethod def setUpTestData(cls): from dojo.models import System_Settings # noqa: PLC0415 - ss = System_Settings.objects.get() + ss = System_Settings.objects.get(no_cache=True) ss.enable_product_tag_inheritance = True ss.save() @@ -593,9 +597,9 @@ def test_baseline_zap_scan_reimport_with_new_findings_v3(self): # Multiple-CWEs feature: +2 import / +2 reimport-no-change (Finding_CWE # store + bulk flush) and +10 reimport-with-new (per-finding reconcile reads # existing Finding_CWE rows for each changed finding). - EXPECTED_ZAP_IMPORT_V2 = 289 - EXPECTED_ZAP_IMPORT_V3 = 313 - EXPECTED_ZAP_REIMPORT_NO_CHANGE_V2 = 76 - EXPECTED_ZAP_REIMPORT_NO_CHANGE_V3 = 88 - EXPECTED_ZAP_REIMPORT_WITH_NEW_V2 = 158 - EXPECTED_ZAP_REIMPORT_WITH_NEW_V3 = 187 + EXPECTED_ZAP_IMPORT_V2 = 294 + EXPECTED_ZAP_IMPORT_V3 = 318 + EXPECTED_ZAP_REIMPORT_NO_CHANGE_V2 = 79 + EXPECTED_ZAP_REIMPORT_NO_CHANGE_V3 = 91 + EXPECTED_ZAP_REIMPORT_WITH_NEW_V2 = 161 + EXPECTED_ZAP_REIMPORT_WITH_NEW_V3 = 190 diff --git a/unittests/test_user_validators.py b/unittests/test_user_validators.py index e91e93550ac..a7b7d91c9e2 100644 --- a/unittests/test_user_validators.py +++ b/unittests/test_user_validators.py @@ -34,7 +34,7 @@ def set_policy( uppercase_character_required=False, non_common_password_required=False, ): - self.system_settings = System_Settings.objects.get() + self.system_settings = System_Settings.objects.get(no_cache=True) self.system_settings.minimum_password_length = minimum_password_length self.system_settings.maximum_password_length = maximum_password_length self.system_settings.number_character_required = number_character_required